nanoarrow 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +3 -0
- data/LICENSE.txt +208 -0
- data/NOTICE.txt +5 -0
- data/README.md +54 -0
- data/ext/nanoarrow/array.c +99 -0
- data/ext/nanoarrow/array_builder.c +268 -0
- data/ext/nanoarrow/array_stream.c +201 -0
- data/ext/nanoarrow/array_view.c +177 -0
- data/ext/nanoarrow/buffer.c +46 -0
- data/ext/nanoarrow/capsule.c +89 -0
- data/ext/nanoarrow/ext.c +23 -0
- data/ext/nanoarrow/ext.h +89 -0
- data/ext/nanoarrow/extconf.rb +7 -0
- data/ext/nanoarrow/mat_array_stream.c +186 -0
- data/ext/nanoarrow/nanoarrow/nanoarrow.h +4482 -0
- data/ext/nanoarrow/nanoarrow.c +4117 -0
- data/ext/nanoarrow/rc.c +34 -0
- data/ext/nanoarrow/schema.c +346 -0
- data/ext/nanoarrow/schema_builder.c +251 -0
- data/ext/nanoarrow/schema_view.c +204 -0
- data/ext/nanoarrow/types.c +48 -0
- data/ext/nanoarrow/utils.c +202 -0
- data/lib/nanoarrow/array.rb +97 -0
- data/lib/nanoarrow/array_builder.rb +115 -0
- data/lib/nanoarrow/array_stream.rb +32 -0
- data/lib/nanoarrow/iterator.rb +113 -0
- data/lib/nanoarrow/schema.rb +85 -0
- data/lib/nanoarrow/types.rb +157 -0
- data/lib/nanoarrow/utils.rb +127 -0
- data/lib/nanoarrow/version.rb +3 -0
- data/lib/nanoarrow.rb +28 -0
- metadata +70 -0
|
@@ -0,0 +1,4482 @@
|
|
|
1
|
+
// Licensed to the Apache Software Foundation (ASF) under one
|
|
2
|
+
// or more contributor license agreements. See the NOTICE file
|
|
3
|
+
// distributed with this work for additional information
|
|
4
|
+
// regarding copyright ownership. The ASF licenses this file
|
|
5
|
+
// to you under the Apache License, Version 2.0 (the
|
|
6
|
+
// "License"); you may not use this file except in compliance
|
|
7
|
+
// with the License. You may obtain a copy of the License at
|
|
8
|
+
//
|
|
9
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
//
|
|
11
|
+
// Unless required by applicable law or agreed to in writing,
|
|
12
|
+
// software distributed under the License is distributed on an
|
|
13
|
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
14
|
+
// KIND, either express or implied. See the License for the
|
|
15
|
+
// specific language governing permissions and limitations
|
|
16
|
+
// under the License.
|
|
17
|
+
|
|
18
|
+
#ifndef NANOARROW_CONFIG_H_INCLUDED
|
|
19
|
+
#define NANOARROW_CONFIG_H_INCLUDED
|
|
20
|
+
|
|
21
|
+
#define NANOARROW_VERSION_MAJOR 0
|
|
22
|
+
#define NANOARROW_VERSION_MINOR 8
|
|
23
|
+
#define NANOARROW_VERSION_PATCH 0
|
|
24
|
+
#define NANOARROW_VERSION "0.8.0"
|
|
25
|
+
|
|
26
|
+
#define NANOARROW_VERSION_INT \
|
|
27
|
+
(NANOARROW_VERSION_MAJOR * 10000 + NANOARROW_VERSION_MINOR * 100 + \
|
|
28
|
+
NANOARROW_VERSION_PATCH)
|
|
29
|
+
|
|
30
|
+
// #define NANOARROW_NAMESPACE YourNamespaceHere
|
|
31
|
+
|
|
32
|
+
#if !defined(NANOARROW_CXX_NAMESPACE)
|
|
33
|
+
#define NANOARROW_CXX_NAMESPACE nanoarrow
|
|
34
|
+
#endif
|
|
35
|
+
|
|
36
|
+
#define NANOARROW_CXX_NAMESPACE_BEGIN namespace NANOARROW_CXX_NAMESPACE {
|
|
37
|
+
#define NANOARROW_CXX_NAMESPACE_END }
|
|
38
|
+
|
|
39
|
+
#endif
|
|
40
|
+
// Licensed to the Apache Software Foundation (ASF) under one
|
|
41
|
+
// or more contributor license agreements. See the NOTICE file
|
|
42
|
+
// distributed with this work for additional information
|
|
43
|
+
// regarding copyright ownership. The ASF licenses this file
|
|
44
|
+
// to you under the Apache License, Version 2.0 (the
|
|
45
|
+
// "License"); you may not use this file except in compliance
|
|
46
|
+
// with the License. You may obtain a copy of the License at
|
|
47
|
+
//
|
|
48
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
49
|
+
//
|
|
50
|
+
// Unless required by applicable law or agreed to in writing,
|
|
51
|
+
// software distributed under the License is distributed on an
|
|
52
|
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
53
|
+
// KIND, either express or implied. See the License for the
|
|
54
|
+
// specific language governing permissions and limitations
|
|
55
|
+
// under the License.
|
|
56
|
+
|
|
57
|
+
#ifndef NANOARROW_NANOARROW_TYPES_H_INCLUDED
|
|
58
|
+
#define NANOARROW_NANOARROW_TYPES_H_INCLUDED
|
|
59
|
+
|
|
60
|
+
#include <stdint.h>
|
|
61
|
+
#include <string.h>
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
#if defined(NANOARROW_DEBUG) && !defined(NANOARROW_PRINT_AND_DIE)
|
|
66
|
+
#include <stdio.h>
|
|
67
|
+
#include <stdlib.h>
|
|
68
|
+
#endif
|
|
69
|
+
|
|
70
|
+
#ifdef __cplusplus
|
|
71
|
+
extern "C" {
|
|
72
|
+
#endif
|
|
73
|
+
|
|
74
|
+
// Extra guard for versions of Arrow without the canonical guard
|
|
75
|
+
#ifndef ARROW_FLAG_DICTIONARY_ORDERED
|
|
76
|
+
|
|
77
|
+
/// \defgroup nanoarrow-arrow-cdata Arrow C Data interface
|
|
78
|
+
///
|
|
79
|
+
/// The Arrow C Data (https://arrow.apache.org/docs/format/CDataInterface.html)
|
|
80
|
+
/// and Arrow C Stream (https://arrow.apache.org/docs/format/CStreamInterface.html)
|
|
81
|
+
/// interfaces are part of the
|
|
82
|
+
/// Arrow Columnar Format specification
|
|
83
|
+
/// (https://arrow.apache.org/docs/format/Columnar.html). See the Arrow documentation for
|
|
84
|
+
/// documentation of these structures.
|
|
85
|
+
///
|
|
86
|
+
/// @{
|
|
87
|
+
|
|
88
|
+
#ifndef ARROW_C_DATA_INTERFACE
|
|
89
|
+
#define ARROW_C_DATA_INTERFACE
|
|
90
|
+
|
|
91
|
+
#define ARROW_FLAG_DICTIONARY_ORDERED 1
|
|
92
|
+
#define ARROW_FLAG_NULLABLE 2
|
|
93
|
+
#define ARROW_FLAG_MAP_KEYS_SORTED 4
|
|
94
|
+
|
|
95
|
+
struct ArrowSchema {
|
|
96
|
+
// Array type description
|
|
97
|
+
const char* format;
|
|
98
|
+
const char* name;
|
|
99
|
+
const char* metadata;
|
|
100
|
+
int64_t flags;
|
|
101
|
+
int64_t n_children;
|
|
102
|
+
struct ArrowSchema** children;
|
|
103
|
+
struct ArrowSchema* dictionary;
|
|
104
|
+
|
|
105
|
+
// Release callback
|
|
106
|
+
void (*release)(struct ArrowSchema*);
|
|
107
|
+
// Opaque producer-specific data
|
|
108
|
+
void* private_data;
|
|
109
|
+
};
|
|
110
|
+
|
|
111
|
+
struct ArrowArray {
|
|
112
|
+
// Array data description
|
|
113
|
+
int64_t length;
|
|
114
|
+
int64_t null_count;
|
|
115
|
+
int64_t offset;
|
|
116
|
+
int64_t n_buffers;
|
|
117
|
+
int64_t n_children;
|
|
118
|
+
const void** buffers;
|
|
119
|
+
struct ArrowArray** children;
|
|
120
|
+
struct ArrowArray* dictionary;
|
|
121
|
+
|
|
122
|
+
// Release callback
|
|
123
|
+
void (*release)(struct ArrowArray*);
|
|
124
|
+
// Opaque producer-specific data
|
|
125
|
+
void* private_data;
|
|
126
|
+
};
|
|
127
|
+
|
|
128
|
+
#endif // ARROW_C_DATA_INTERFACE
|
|
129
|
+
|
|
130
|
+
#ifndef ARROW_C_STREAM_INTERFACE
|
|
131
|
+
#define ARROW_C_STREAM_INTERFACE
|
|
132
|
+
|
|
133
|
+
struct ArrowArrayStream {
|
|
134
|
+
// Callback to get the stream type
|
|
135
|
+
// (will be the same for all arrays in the stream).
|
|
136
|
+
//
|
|
137
|
+
// Return value: 0 if successful, an `errno`-compatible error code otherwise.
|
|
138
|
+
//
|
|
139
|
+
// If successful, the ArrowSchema must be released independently from the stream.
|
|
140
|
+
int (*get_schema)(struct ArrowArrayStream*, struct ArrowSchema* out);
|
|
141
|
+
|
|
142
|
+
// Callback to get the next array
|
|
143
|
+
// (if no error and the array is released, the stream has ended)
|
|
144
|
+
//
|
|
145
|
+
// Return value: 0 if successful, an `errno`-compatible error code otherwise.
|
|
146
|
+
//
|
|
147
|
+
// If successful, the ArrowArray must be released independently from the stream.
|
|
148
|
+
int (*get_next)(struct ArrowArrayStream*, struct ArrowArray* out);
|
|
149
|
+
|
|
150
|
+
// Callback to get optional detailed error information.
|
|
151
|
+
// This must only be called if the last stream operation failed
|
|
152
|
+
// with a non-0 return code.
|
|
153
|
+
//
|
|
154
|
+
// Return value: pointer to a null-terminated character array describing
|
|
155
|
+
// the last error, or NULL if no description is available.
|
|
156
|
+
//
|
|
157
|
+
// The returned pointer is only valid until the next operation on this stream
|
|
158
|
+
// (including release).
|
|
159
|
+
const char* (*get_last_error)(struct ArrowArrayStream*);
|
|
160
|
+
|
|
161
|
+
// Release callback: release the stream's own resources.
|
|
162
|
+
// Note that arrays returned by `get_next` must be individually released.
|
|
163
|
+
void (*release)(struct ArrowArrayStream*);
|
|
164
|
+
|
|
165
|
+
// Opaque producer-specific data
|
|
166
|
+
void* private_data;
|
|
167
|
+
};
|
|
168
|
+
|
|
169
|
+
#endif // ARROW_C_STREAM_INTERFACE
|
|
170
|
+
#endif // ARROW_FLAG_DICTIONARY_ORDERED
|
|
171
|
+
|
|
172
|
+
/// @}
|
|
173
|
+
|
|
174
|
+
// Utility macros
|
|
175
|
+
#define _NANOARROW_CONCAT(x, y) x##y
|
|
176
|
+
#define _NANOARROW_MAKE_NAME(x, y) _NANOARROW_CONCAT(x, y)
|
|
177
|
+
|
|
178
|
+
#define _NANOARROW_RETURN_NOT_OK_IMPL(NAME, EXPR) \
|
|
179
|
+
do { \
|
|
180
|
+
const int NAME = (EXPR); \
|
|
181
|
+
if (NAME) return NAME; \
|
|
182
|
+
} while (0)
|
|
183
|
+
|
|
184
|
+
#define _NANOARROW_CHECK_RANGE(x_, min_, max_) \
|
|
185
|
+
NANOARROW_RETURN_NOT_OK((x_ >= min_ && x_ <= max_) ? NANOARROW_OK : EINVAL)
|
|
186
|
+
|
|
187
|
+
#define _NANOARROW_CHECK_UPPER_LIMIT(x_, max_) \
|
|
188
|
+
NANOARROW_RETURN_NOT_OK((x_ <= max_) ? NANOARROW_OK : EINVAL)
|
|
189
|
+
|
|
190
|
+
#if defined(NANOARROW_DEBUG)
|
|
191
|
+
#define _NANOARROW_RETURN_NOT_OK_WITH_ERROR_IMPL(NAME, EXPR, ERROR_PTR_EXPR, EXPR_STR) \
|
|
192
|
+
do { \
|
|
193
|
+
const int NAME = (EXPR); \
|
|
194
|
+
if (NAME) { \
|
|
195
|
+
ArrowErrorSet((ERROR_PTR_EXPR), "%s failed with errno %d\n* %s:%d", EXPR_STR, \
|
|
196
|
+
NAME, __FILE__, __LINE__); \
|
|
197
|
+
return NAME; \
|
|
198
|
+
} \
|
|
199
|
+
} while (0)
|
|
200
|
+
#else
|
|
201
|
+
#define _NANOARROW_RETURN_NOT_OK_WITH_ERROR_IMPL(NAME, EXPR, ERROR_PTR_EXPR, EXPR_STR) \
|
|
202
|
+
do { \
|
|
203
|
+
const int NAME = (EXPR); \
|
|
204
|
+
if (NAME) { \
|
|
205
|
+
ArrowErrorSet((ERROR_PTR_EXPR), "%s failed with errno %d", EXPR_STR, NAME); \
|
|
206
|
+
return NAME; \
|
|
207
|
+
} \
|
|
208
|
+
} while (0)
|
|
209
|
+
#endif
|
|
210
|
+
|
|
211
|
+
#if defined(NANOARROW_DEBUG)
|
|
212
|
+
// For checking ArrowErrorSet() calls for valid printf format strings/arguments
|
|
213
|
+
// If using mingw's c99-compliant printf, we need a different format-checking attribute
|
|
214
|
+
#if defined(__USE_MINGW_ANSI_STDIO) && defined(__MINGW_PRINTF_FORMAT)
|
|
215
|
+
#define NANOARROW_CHECK_PRINTF_ATTRIBUTE \
|
|
216
|
+
__attribute__((format(__MINGW_PRINTF_FORMAT, 2, 3)))
|
|
217
|
+
#elif defined(__GNUC__)
|
|
218
|
+
#define NANOARROW_CHECK_PRINTF_ATTRIBUTE __attribute__((format(printf, 2, 3)))
|
|
219
|
+
#else
|
|
220
|
+
#define NANOARROW_CHECK_PRINTF_ATTRIBUTE
|
|
221
|
+
#endif
|
|
222
|
+
|
|
223
|
+
// For checking calls to functions that return ArrowErrorCode
|
|
224
|
+
#if defined(__GNUC__) && (__GNUC__ >= 4)
|
|
225
|
+
#define NANOARROW_CHECK_RETURN_ATTRIBUTE __attribute__((warn_unused_result))
|
|
226
|
+
#elif defined(_MSC_VER) && (_MSC_VER >= 1700)
|
|
227
|
+
#define NANOARROW_CHECK_RETURN_ATTRIBUTE _Check_return_
|
|
228
|
+
#else
|
|
229
|
+
#define NANOARROW_CHECK_RETURN_ATTRIBUTE
|
|
230
|
+
#endif
|
|
231
|
+
|
|
232
|
+
#else
|
|
233
|
+
#define NANOARROW_CHECK_RETURN_ATTRIBUTE
|
|
234
|
+
#define NANOARROW_CHECK_PRINTF_ATTRIBUTE
|
|
235
|
+
#endif
|
|
236
|
+
|
|
237
|
+
#define NANOARROW_UNUSED(x) (void)(x)
|
|
238
|
+
|
|
239
|
+
/// \brief Return code for success.
|
|
240
|
+
/// \ingroup nanoarrow-errors
|
|
241
|
+
#define NANOARROW_OK 0
|
|
242
|
+
|
|
243
|
+
/// \brief Represents an errno-compatible error code
|
|
244
|
+
/// \ingroup nanoarrow-errors
|
|
245
|
+
typedef int ArrowErrorCode;
|
|
246
|
+
|
|
247
|
+
#if defined(NANOARROW_DEBUG)
|
|
248
|
+
#define ArrowErrorCode NANOARROW_CHECK_RETURN_ATTRIBUTE ArrowErrorCode
|
|
249
|
+
#endif
|
|
250
|
+
|
|
251
|
+
/// \brief Flags supported by ArrowSchemaViewInit()
|
|
252
|
+
/// \ingroup nanoarrow-schema-view
|
|
253
|
+
#define NANOARROW_FLAG_ALL_SUPPORTED \
|
|
254
|
+
(ARROW_FLAG_DICTIONARY_ORDERED | ARROW_FLAG_NULLABLE | ARROW_FLAG_MAP_KEYS_SORTED)
|
|
255
|
+
|
|
256
|
+
/// \brief Error type containing a UTF-8 encoded message.
|
|
257
|
+
/// \ingroup nanoarrow-errors
|
|
258
|
+
struct ArrowError {
|
|
259
|
+
/// \brief A character buffer with space for an error message.
|
|
260
|
+
char message[1024];
|
|
261
|
+
};
|
|
262
|
+
|
|
263
|
+
/// \brief Ensure an ArrowError is null-terminated by zeroing the first character.
|
|
264
|
+
/// \ingroup nanoarrow-errors
|
|
265
|
+
///
|
|
266
|
+
/// If error is NULL, this function does nothing.
|
|
267
|
+
static inline void ArrowErrorInit(struct ArrowError* error) {
|
|
268
|
+
if (error != NULL) {
|
|
269
|
+
error->message[0] = '\0';
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/// \brief Get the contents of an error
|
|
274
|
+
/// \ingroup nanoarrow-errors
|
|
275
|
+
///
|
|
276
|
+
/// If error is NULL, returns "", or returns the contents of the error message
|
|
277
|
+
/// otherwise.
|
|
278
|
+
static inline const char* ArrowErrorMessage(struct ArrowError* error) {
|
|
279
|
+
if (error == NULL) {
|
|
280
|
+
return "";
|
|
281
|
+
} else {
|
|
282
|
+
return error->message;
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
/// \brief Set the contents of an error from an existing null-terminated string
|
|
287
|
+
/// \ingroup nanoarrow-errors
|
|
288
|
+
///
|
|
289
|
+
/// If error is NULL, this function does nothing.
|
|
290
|
+
static inline void ArrowErrorSetString(struct ArrowError* error, const char* src) {
|
|
291
|
+
if (error == NULL) {
|
|
292
|
+
return;
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
int64_t src_len = strlen(src);
|
|
296
|
+
if (src_len >= ((int64_t)sizeof(error->message))) {
|
|
297
|
+
memcpy(error->message, src, sizeof(error->message) - 1);
|
|
298
|
+
error->message[sizeof(error->message) - 1] = '\0';
|
|
299
|
+
} else {
|
|
300
|
+
memcpy(error->message, src, src_len);
|
|
301
|
+
error->message[src_len] = '\0';
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
/// \brief Check the result of an expression and return it if not NANOARROW_OK
|
|
306
|
+
/// \ingroup nanoarrow-errors
|
|
307
|
+
#define NANOARROW_RETURN_NOT_OK(EXPR) \
|
|
308
|
+
_NANOARROW_RETURN_NOT_OK_IMPL(_NANOARROW_MAKE_NAME(errno_status_, __COUNTER__), EXPR)
|
|
309
|
+
|
|
310
|
+
/// \brief Check the result of an expression and return it if not NANOARROW_OK,
|
|
311
|
+
/// adding an auto-generated message to an ArrowError.
|
|
312
|
+
/// \ingroup nanoarrow-errors
|
|
313
|
+
///
|
|
314
|
+
/// This macro is used to ensure that functions that accept an ArrowError
|
|
315
|
+
/// as input always set its message when returning an error code (e.g., when calling
|
|
316
|
+
/// a nanoarrow function that does *not* accept ArrowError).
|
|
317
|
+
#define NANOARROW_RETURN_NOT_OK_WITH_ERROR(EXPR, ERROR_EXPR) \
|
|
318
|
+
_NANOARROW_RETURN_NOT_OK_WITH_ERROR_IMPL( \
|
|
319
|
+
_NANOARROW_MAKE_NAME(errno_status_, __COUNTER__), EXPR, ERROR_EXPR, #EXPR)
|
|
320
|
+
|
|
321
|
+
#if defined(NANOARROW_DEBUG) && !defined(NANOARROW_PRINT_AND_DIE)
|
|
322
|
+
#define NANOARROW_PRINT_AND_DIE(VALUE, EXPR_STR) \
|
|
323
|
+
do { \
|
|
324
|
+
fprintf(stderr, "%s failed with code %d\n* %s:%d\n", EXPR_STR, (int)(VALUE), \
|
|
325
|
+
__FILE__, (int)__LINE__); \
|
|
326
|
+
abort(); \
|
|
327
|
+
} while (0)
|
|
328
|
+
#endif
|
|
329
|
+
|
|
330
|
+
#if defined(NANOARROW_DEBUG)
|
|
331
|
+
#define _NANOARROW_ASSERT_OK_IMPL(NAME, EXPR, EXPR_STR) \
|
|
332
|
+
do { \
|
|
333
|
+
const int NAME = (EXPR); \
|
|
334
|
+
if (NAME) NANOARROW_PRINT_AND_DIE(NAME, EXPR_STR); \
|
|
335
|
+
} while (0)
|
|
336
|
+
|
|
337
|
+
/// \brief Assert that an expression's value is NANOARROW_OK
|
|
338
|
+
/// \ingroup nanoarrow-errors
|
|
339
|
+
///
|
|
340
|
+
/// If nanoarrow was built in debug mode (i.e., defined(NANOARROW_DEBUG) is true),
|
|
341
|
+
/// print a message to stderr and abort. If nanoarrow was built in release mode,
|
|
342
|
+
/// this statement has no effect. You can customize fatal error behaviour
|
|
343
|
+
/// be defining the NANOARROW_PRINT_AND_DIE macro before including nanoarrow.h
|
|
344
|
+
/// This macro is provided as a convenience for users and is not used internally.
|
|
345
|
+
#define NANOARROW_ASSERT_OK(EXPR) \
|
|
346
|
+
_NANOARROW_ASSERT_OK_IMPL(_NANOARROW_MAKE_NAME(errno_status_, __COUNTER__), EXPR, #EXPR)
|
|
347
|
+
|
|
348
|
+
#define _NANOARROW_DCHECK_IMPL(EXPR, EXPR_STR) \
|
|
349
|
+
do { \
|
|
350
|
+
if (!(EXPR)) NANOARROW_PRINT_AND_DIE(-1, EXPR_STR); \
|
|
351
|
+
} while (0)
|
|
352
|
+
|
|
353
|
+
#define NANOARROW_DCHECK(EXPR) _NANOARROW_DCHECK_IMPL(EXPR, #EXPR)
|
|
354
|
+
#else
|
|
355
|
+
#define NANOARROW_ASSERT_OK(EXPR) (void)(EXPR)
|
|
356
|
+
#define NANOARROW_DCHECK(EXPR)
|
|
357
|
+
#endif
|
|
358
|
+
|
|
359
|
+
static inline void ArrowSchemaMove(struct ArrowSchema* src, struct ArrowSchema* dst) {
|
|
360
|
+
NANOARROW_DCHECK(src != NULL);
|
|
361
|
+
NANOARROW_DCHECK(dst != NULL);
|
|
362
|
+
|
|
363
|
+
memcpy(dst, src, sizeof(struct ArrowSchema));
|
|
364
|
+
src->release = NULL;
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
static inline void ArrowSchemaRelease(struct ArrowSchema* schema) {
|
|
368
|
+
NANOARROW_DCHECK(schema != NULL);
|
|
369
|
+
schema->release(schema);
|
|
370
|
+
NANOARROW_DCHECK(schema->release == NULL);
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
static inline void ArrowArrayMove(struct ArrowArray* src, struct ArrowArray* dst) {
|
|
374
|
+
NANOARROW_DCHECK(src != NULL);
|
|
375
|
+
NANOARROW_DCHECK(dst != NULL);
|
|
376
|
+
|
|
377
|
+
memcpy(dst, src, sizeof(struct ArrowArray));
|
|
378
|
+
src->release = NULL;
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
static inline void ArrowArrayRelease(struct ArrowArray* array) {
|
|
382
|
+
NANOARROW_DCHECK(array != NULL);
|
|
383
|
+
array->release(array);
|
|
384
|
+
NANOARROW_DCHECK(array->release == NULL);
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
static inline void ArrowArrayStreamMove(struct ArrowArrayStream* src,
|
|
388
|
+
struct ArrowArrayStream* dst) {
|
|
389
|
+
NANOARROW_DCHECK(src != NULL);
|
|
390
|
+
NANOARROW_DCHECK(dst != NULL);
|
|
391
|
+
|
|
392
|
+
memcpy(dst, src, sizeof(struct ArrowArrayStream));
|
|
393
|
+
src->release = NULL;
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
static inline const char* ArrowArrayStreamGetLastError(
|
|
397
|
+
struct ArrowArrayStream* array_stream) {
|
|
398
|
+
NANOARROW_DCHECK(array_stream != NULL);
|
|
399
|
+
|
|
400
|
+
const char* value = array_stream->get_last_error(array_stream);
|
|
401
|
+
if (value == NULL) {
|
|
402
|
+
return "";
|
|
403
|
+
} else {
|
|
404
|
+
return value;
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
static inline ArrowErrorCode ArrowArrayStreamGetSchema(
|
|
409
|
+
struct ArrowArrayStream* array_stream, struct ArrowSchema* out,
|
|
410
|
+
struct ArrowError* error) {
|
|
411
|
+
NANOARROW_DCHECK(array_stream != NULL);
|
|
412
|
+
|
|
413
|
+
int result = array_stream->get_schema(array_stream, out);
|
|
414
|
+
if (result != NANOARROW_OK && error != NULL) {
|
|
415
|
+
ArrowErrorSetString(error, ArrowArrayStreamGetLastError(array_stream));
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
return result;
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
static inline ArrowErrorCode ArrowArrayStreamGetNext(
|
|
422
|
+
struct ArrowArrayStream* array_stream, struct ArrowArray* out,
|
|
423
|
+
struct ArrowError* error) {
|
|
424
|
+
NANOARROW_DCHECK(array_stream != NULL);
|
|
425
|
+
|
|
426
|
+
int result = array_stream->get_next(array_stream, out);
|
|
427
|
+
if (result != NANOARROW_OK && error != NULL) {
|
|
428
|
+
ArrowErrorSetString(error, ArrowArrayStreamGetLastError(array_stream));
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
return result;
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
static inline void ArrowArrayStreamRelease(struct ArrowArrayStream* array_stream) {
|
|
435
|
+
NANOARROW_DCHECK(array_stream != NULL);
|
|
436
|
+
array_stream->release(array_stream);
|
|
437
|
+
NANOARROW_DCHECK(array_stream->release == NULL);
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
static char _ArrowIsLittleEndian(void) {
|
|
441
|
+
uint32_t check = 1;
|
|
442
|
+
char first_byte;
|
|
443
|
+
memcpy(&first_byte, &check, sizeof(char));
|
|
444
|
+
return first_byte;
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
/// \brief Arrow type enumerator
|
|
448
|
+
/// \ingroup nanoarrow-utils
|
|
449
|
+
///
|
|
450
|
+
/// These names are intended to map to the corresponding arrow::Type::type
|
|
451
|
+
/// enumerator; however, the numeric values are specifically not equal
|
|
452
|
+
/// (i.e., do not rely on numeric comparison).
|
|
453
|
+
enum ArrowType {
|
|
454
|
+
NANOARROW_TYPE_UNINITIALIZED = 0,
|
|
455
|
+
NANOARROW_TYPE_NA = 1,
|
|
456
|
+
NANOARROW_TYPE_BOOL,
|
|
457
|
+
NANOARROW_TYPE_UINT8,
|
|
458
|
+
NANOARROW_TYPE_INT8,
|
|
459
|
+
NANOARROW_TYPE_UINT16,
|
|
460
|
+
NANOARROW_TYPE_INT16,
|
|
461
|
+
NANOARROW_TYPE_UINT32,
|
|
462
|
+
NANOARROW_TYPE_INT32,
|
|
463
|
+
NANOARROW_TYPE_UINT64,
|
|
464
|
+
NANOARROW_TYPE_INT64,
|
|
465
|
+
NANOARROW_TYPE_HALF_FLOAT,
|
|
466
|
+
NANOARROW_TYPE_FLOAT,
|
|
467
|
+
NANOARROW_TYPE_DOUBLE,
|
|
468
|
+
NANOARROW_TYPE_STRING,
|
|
469
|
+
NANOARROW_TYPE_BINARY,
|
|
470
|
+
NANOARROW_TYPE_FIXED_SIZE_BINARY,
|
|
471
|
+
NANOARROW_TYPE_DATE32,
|
|
472
|
+
NANOARROW_TYPE_DATE64,
|
|
473
|
+
NANOARROW_TYPE_TIMESTAMP,
|
|
474
|
+
NANOARROW_TYPE_TIME32,
|
|
475
|
+
NANOARROW_TYPE_TIME64,
|
|
476
|
+
NANOARROW_TYPE_INTERVAL_MONTHS,
|
|
477
|
+
NANOARROW_TYPE_INTERVAL_DAY_TIME,
|
|
478
|
+
NANOARROW_TYPE_DECIMAL128,
|
|
479
|
+
NANOARROW_TYPE_DECIMAL256,
|
|
480
|
+
NANOARROW_TYPE_LIST,
|
|
481
|
+
NANOARROW_TYPE_STRUCT,
|
|
482
|
+
NANOARROW_TYPE_SPARSE_UNION,
|
|
483
|
+
NANOARROW_TYPE_DENSE_UNION,
|
|
484
|
+
NANOARROW_TYPE_DICTIONARY,
|
|
485
|
+
NANOARROW_TYPE_MAP,
|
|
486
|
+
NANOARROW_TYPE_EXTENSION,
|
|
487
|
+
NANOARROW_TYPE_FIXED_SIZE_LIST,
|
|
488
|
+
NANOARROW_TYPE_DURATION,
|
|
489
|
+
NANOARROW_TYPE_LARGE_STRING,
|
|
490
|
+
NANOARROW_TYPE_LARGE_BINARY,
|
|
491
|
+
NANOARROW_TYPE_LARGE_LIST,
|
|
492
|
+
NANOARROW_TYPE_INTERVAL_MONTH_DAY_NANO,
|
|
493
|
+
NANOARROW_TYPE_RUN_END_ENCODED,
|
|
494
|
+
NANOARROW_TYPE_BINARY_VIEW,
|
|
495
|
+
NANOARROW_TYPE_STRING_VIEW,
|
|
496
|
+
NANOARROW_TYPE_DECIMAL32,
|
|
497
|
+
NANOARROW_TYPE_DECIMAL64,
|
|
498
|
+
NANOARROW_TYPE_LIST_VIEW,
|
|
499
|
+
NANOARROW_TYPE_LARGE_LIST_VIEW,
|
|
500
|
+
};
|
|
501
|
+
|
|
502
|
+
/// \brief Get a string value of an enum ArrowType value
|
|
503
|
+
/// \ingroup nanoarrow-utils
|
|
504
|
+
///
|
|
505
|
+
/// Returns NULL for invalid values for type
|
|
506
|
+
static inline const char* ArrowTypeString(enum ArrowType type);
|
|
507
|
+
|
|
508
|
+
static inline const char* ArrowTypeString(enum ArrowType type) {
|
|
509
|
+
switch (type) {
|
|
510
|
+
case NANOARROW_TYPE_NA:
|
|
511
|
+
return "na";
|
|
512
|
+
case NANOARROW_TYPE_BOOL:
|
|
513
|
+
return "bool";
|
|
514
|
+
case NANOARROW_TYPE_UINT8:
|
|
515
|
+
return "uint8";
|
|
516
|
+
case NANOARROW_TYPE_INT8:
|
|
517
|
+
return "int8";
|
|
518
|
+
case NANOARROW_TYPE_UINT16:
|
|
519
|
+
return "uint16";
|
|
520
|
+
case NANOARROW_TYPE_INT16:
|
|
521
|
+
return "int16";
|
|
522
|
+
case NANOARROW_TYPE_UINT32:
|
|
523
|
+
return "uint32";
|
|
524
|
+
case NANOARROW_TYPE_INT32:
|
|
525
|
+
return "int32";
|
|
526
|
+
case NANOARROW_TYPE_UINT64:
|
|
527
|
+
return "uint64";
|
|
528
|
+
case NANOARROW_TYPE_INT64:
|
|
529
|
+
return "int64";
|
|
530
|
+
case NANOARROW_TYPE_HALF_FLOAT:
|
|
531
|
+
return "half_float";
|
|
532
|
+
case NANOARROW_TYPE_FLOAT:
|
|
533
|
+
return "float";
|
|
534
|
+
case NANOARROW_TYPE_DOUBLE:
|
|
535
|
+
return "double";
|
|
536
|
+
case NANOARROW_TYPE_STRING:
|
|
537
|
+
return "string";
|
|
538
|
+
case NANOARROW_TYPE_BINARY:
|
|
539
|
+
return "binary";
|
|
540
|
+
case NANOARROW_TYPE_FIXED_SIZE_BINARY:
|
|
541
|
+
return "fixed_size_binary";
|
|
542
|
+
case NANOARROW_TYPE_DATE32:
|
|
543
|
+
return "date32";
|
|
544
|
+
case NANOARROW_TYPE_DATE64:
|
|
545
|
+
return "date64";
|
|
546
|
+
case NANOARROW_TYPE_TIMESTAMP:
|
|
547
|
+
return "timestamp";
|
|
548
|
+
case NANOARROW_TYPE_TIME32:
|
|
549
|
+
return "time32";
|
|
550
|
+
case NANOARROW_TYPE_TIME64:
|
|
551
|
+
return "time64";
|
|
552
|
+
case NANOARROW_TYPE_INTERVAL_MONTHS:
|
|
553
|
+
return "interval_months";
|
|
554
|
+
case NANOARROW_TYPE_INTERVAL_DAY_TIME:
|
|
555
|
+
return "interval_day_time";
|
|
556
|
+
case NANOARROW_TYPE_DECIMAL32:
|
|
557
|
+
return "decimal32";
|
|
558
|
+
case NANOARROW_TYPE_DECIMAL64:
|
|
559
|
+
return "decimal64";
|
|
560
|
+
case NANOARROW_TYPE_DECIMAL128:
|
|
561
|
+
return "decimal128";
|
|
562
|
+
case NANOARROW_TYPE_DECIMAL256:
|
|
563
|
+
return "decimal256";
|
|
564
|
+
case NANOARROW_TYPE_LIST:
|
|
565
|
+
return "list";
|
|
566
|
+
case NANOARROW_TYPE_STRUCT:
|
|
567
|
+
return "struct";
|
|
568
|
+
case NANOARROW_TYPE_SPARSE_UNION:
|
|
569
|
+
return "sparse_union";
|
|
570
|
+
case NANOARROW_TYPE_DENSE_UNION:
|
|
571
|
+
return "dense_union";
|
|
572
|
+
case NANOARROW_TYPE_DICTIONARY:
|
|
573
|
+
return "dictionary";
|
|
574
|
+
case NANOARROW_TYPE_MAP:
|
|
575
|
+
return "map";
|
|
576
|
+
case NANOARROW_TYPE_EXTENSION:
|
|
577
|
+
return "extension";
|
|
578
|
+
case NANOARROW_TYPE_FIXED_SIZE_LIST:
|
|
579
|
+
return "fixed_size_list";
|
|
580
|
+
case NANOARROW_TYPE_DURATION:
|
|
581
|
+
return "duration";
|
|
582
|
+
case NANOARROW_TYPE_LARGE_STRING:
|
|
583
|
+
return "large_string";
|
|
584
|
+
case NANOARROW_TYPE_LARGE_BINARY:
|
|
585
|
+
return "large_binary";
|
|
586
|
+
case NANOARROW_TYPE_LARGE_LIST:
|
|
587
|
+
return "large_list";
|
|
588
|
+
case NANOARROW_TYPE_INTERVAL_MONTH_DAY_NANO:
|
|
589
|
+
return "interval_month_day_nano";
|
|
590
|
+
case NANOARROW_TYPE_RUN_END_ENCODED:
|
|
591
|
+
return "run_end_encoded";
|
|
592
|
+
case NANOARROW_TYPE_BINARY_VIEW:
|
|
593
|
+
return "binary_view";
|
|
594
|
+
case NANOARROW_TYPE_STRING_VIEW:
|
|
595
|
+
return "string_view";
|
|
596
|
+
case NANOARROW_TYPE_LIST_VIEW:
|
|
597
|
+
return "list_view";
|
|
598
|
+
case NANOARROW_TYPE_LARGE_LIST_VIEW:
|
|
599
|
+
return "large_list_view";
|
|
600
|
+
default:
|
|
601
|
+
return NULL;
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
/// \brief Arrow time unit enumerator
|
|
606
|
+
/// \ingroup nanoarrow-utils
|
|
607
|
+
///
|
|
608
|
+
/// These names and values map to the corresponding arrow::TimeUnit::type
|
|
609
|
+
/// enumerator.
|
|
610
|
+
enum ArrowTimeUnit {
|
|
611
|
+
NANOARROW_TIME_UNIT_SECOND = 0,
|
|
612
|
+
NANOARROW_TIME_UNIT_MILLI = 1,
|
|
613
|
+
NANOARROW_TIME_UNIT_MICRO = 2,
|
|
614
|
+
NANOARROW_TIME_UNIT_NANO = 3
|
|
615
|
+
};
|
|
616
|
+
|
|
617
|
+
/// \brief Validation level enumerator
|
|
618
|
+
/// \ingroup nanoarrow-array
|
|
619
|
+
enum ArrowValidationLevel {
|
|
620
|
+
/// \brief Do not validate buffer sizes or content.
|
|
621
|
+
NANOARROW_VALIDATION_LEVEL_NONE = 0,
|
|
622
|
+
|
|
623
|
+
/// \brief Validate buffer sizes that depend on array length but do not validate buffer
|
|
624
|
+
/// sizes that depend on buffer data access.
|
|
625
|
+
NANOARROW_VALIDATION_LEVEL_MINIMAL = 1,
|
|
626
|
+
|
|
627
|
+
/// \brief Validate all buffer sizes, including those that require buffer data access,
|
|
628
|
+
/// but do not perform any checks that are O(1) along the length of the buffers.
|
|
629
|
+
NANOARROW_VALIDATION_LEVEL_DEFAULT = 2,
|
|
630
|
+
|
|
631
|
+
/// \brief Validate all buffer sizes and all buffer content. This is useful in the
|
|
632
|
+
/// context of untrusted input or input that may have been corrupted in transit.
|
|
633
|
+
NANOARROW_VALIDATION_LEVEL_FULL = 3
|
|
634
|
+
};
|
|
635
|
+
|
|
636
|
+
/// \brief Comparison level enumerator
|
|
637
|
+
/// \ingroup nanoarrow-utils
|
|
638
|
+
enum ArrowCompareLevel {
|
|
639
|
+
/// \brief Consider arrays equal if buffers contain identical content
|
|
640
|
+
/// and have identical offset, null count, and length. Note that this is
|
|
641
|
+
/// a much stricter check than logical equality, which would take into
|
|
642
|
+
/// account potentially different content of null slots, arrays with a
|
|
643
|
+
/// non-zero offset, and other considerations.
|
|
644
|
+
NANOARROW_COMPARE_IDENTICAL,
|
|
645
|
+
};
|
|
646
|
+
|
|
647
|
+
/// \brief Get a string value of an enum ArrowTimeUnit value
|
|
648
|
+
/// \ingroup nanoarrow-utils
|
|
649
|
+
///
|
|
650
|
+
/// Returns NULL for invalid values for time_unit
|
|
651
|
+
static inline const char* ArrowTimeUnitString(enum ArrowTimeUnit time_unit);
|
|
652
|
+
|
|
653
|
+
static inline const char* ArrowTimeUnitString(enum ArrowTimeUnit time_unit) {
|
|
654
|
+
switch (time_unit) {
|
|
655
|
+
case NANOARROW_TIME_UNIT_SECOND:
|
|
656
|
+
return "s";
|
|
657
|
+
case NANOARROW_TIME_UNIT_MILLI:
|
|
658
|
+
return "ms";
|
|
659
|
+
case NANOARROW_TIME_UNIT_MICRO:
|
|
660
|
+
return "us";
|
|
661
|
+
case NANOARROW_TIME_UNIT_NANO:
|
|
662
|
+
return "ns";
|
|
663
|
+
default:
|
|
664
|
+
return NULL;
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
/// \brief Functional types of buffers as described in the Arrow Columnar Specification
|
|
669
|
+
/// \ingroup nanoarrow-array-view
|
|
670
|
+
enum ArrowBufferType {
|
|
671
|
+
NANOARROW_BUFFER_TYPE_NONE,
|
|
672
|
+
NANOARROW_BUFFER_TYPE_VALIDITY,
|
|
673
|
+
NANOARROW_BUFFER_TYPE_TYPE_ID,
|
|
674
|
+
NANOARROW_BUFFER_TYPE_UNION_OFFSET,
|
|
675
|
+
NANOARROW_BUFFER_TYPE_DATA_OFFSET,
|
|
676
|
+
NANOARROW_BUFFER_TYPE_DATA,
|
|
677
|
+
NANOARROW_BUFFER_TYPE_VARIADIC_DATA,
|
|
678
|
+
NANOARROW_BUFFER_TYPE_VARIADIC_SIZE,
|
|
679
|
+
NANOARROW_BUFFER_TYPE_VIEW_OFFSET,
|
|
680
|
+
NANOARROW_BUFFER_TYPE_SIZE,
|
|
681
|
+
};
|
|
682
|
+
|
|
683
|
+
/// \brief The maximum number of fixed buffers in an ArrowArrayView or ArrowLayout
|
|
684
|
+
/// \ingroup nanoarrow-array-view
|
|
685
|
+
#define NANOARROW_MAX_FIXED_BUFFERS 3
|
|
686
|
+
|
|
687
|
+
/// \brief An non-owning view of a string
|
|
688
|
+
/// \ingroup nanoarrow-utils
|
|
689
|
+
struct ArrowStringView {
|
|
690
|
+
/// \brief A pointer to the start of the string
|
|
691
|
+
///
|
|
692
|
+
/// If size_bytes is 0, this value may be NULL.
|
|
693
|
+
const char* data;
|
|
694
|
+
|
|
695
|
+
/// \brief The size of the string in bytes,
|
|
696
|
+
///
|
|
697
|
+
/// (Not including the null terminator.)
|
|
698
|
+
int64_t size_bytes;
|
|
699
|
+
};
|
|
700
|
+
|
|
701
|
+
/// \brief Return a view of a const C string
|
|
702
|
+
/// \ingroup nanoarrow-utils
|
|
703
|
+
static inline struct ArrowStringView ArrowCharView(const char* value);
|
|
704
|
+
|
|
705
|
+
static inline struct ArrowStringView ArrowCharView(const char* value) {
|
|
706
|
+
struct ArrowStringView out;
|
|
707
|
+
|
|
708
|
+
out.data = value;
|
|
709
|
+
if (value) {
|
|
710
|
+
out.size_bytes = (int64_t)strlen(value);
|
|
711
|
+
} else {
|
|
712
|
+
out.size_bytes = 0;
|
|
713
|
+
}
|
|
714
|
+
|
|
715
|
+
return out;
|
|
716
|
+
}
|
|
717
|
+
|
|
718
|
+
union ArrowBufferViewData {
|
|
719
|
+
const void* data;
|
|
720
|
+
const int8_t* as_int8;
|
|
721
|
+
const uint8_t* as_uint8;
|
|
722
|
+
const int16_t* as_int16;
|
|
723
|
+
const uint16_t* as_uint16;
|
|
724
|
+
const int32_t* as_int32;
|
|
725
|
+
const uint32_t* as_uint32;
|
|
726
|
+
const int64_t* as_int64;
|
|
727
|
+
const uint64_t* as_uint64;
|
|
728
|
+
const double* as_double;
|
|
729
|
+
const float* as_float;
|
|
730
|
+
const char* as_char;
|
|
731
|
+
const union ArrowBinaryView* as_binary_view;
|
|
732
|
+
};
|
|
733
|
+
|
|
734
|
+
/// \brief An non-owning view of a buffer
|
|
735
|
+
/// \ingroup nanoarrow-utils
|
|
736
|
+
struct ArrowBufferView {
|
|
737
|
+
/// \brief A pointer to the start of the buffer
|
|
738
|
+
///
|
|
739
|
+
/// If size_bytes is 0, this value may be NULL.
|
|
740
|
+
union ArrowBufferViewData data;
|
|
741
|
+
|
|
742
|
+
/// \brief The size of the buffer in bytes
|
|
743
|
+
int64_t size_bytes;
|
|
744
|
+
};
|
|
745
|
+
|
|
746
|
+
/// \brief Array buffer allocation and deallocation
|
|
747
|
+
/// \ingroup nanoarrow-buffer
|
|
748
|
+
///
|
|
749
|
+
/// Container for allocate, reallocate, and free methods that can be used
|
|
750
|
+
/// to customize allocation and deallocation of buffers when constructing
|
|
751
|
+
/// an ArrowArray.
|
|
752
|
+
struct ArrowBufferAllocator {
|
|
753
|
+
/// \brief Reallocate a buffer or return NULL if it cannot be reallocated
|
|
754
|
+
uint8_t* (*reallocate)(struct ArrowBufferAllocator* allocator, uint8_t* ptr,
|
|
755
|
+
int64_t old_size, int64_t new_size);
|
|
756
|
+
|
|
757
|
+
/// \brief Deallocate a buffer allocated by this allocator
|
|
758
|
+
void (*free)(struct ArrowBufferAllocator* allocator, uint8_t* ptr, int64_t size);
|
|
759
|
+
|
|
760
|
+
/// \brief Opaque data specific to the allocator
|
|
761
|
+
void* private_data;
|
|
762
|
+
};
|
|
763
|
+
|
|
764
|
+
typedef void (*ArrowBufferDeallocatorCallback)(struct ArrowBufferAllocator* allocator,
|
|
765
|
+
uint8_t* ptr, int64_t size);
|
|
766
|
+
|
|
767
|
+
/// \brief An owning mutable view of a buffer
|
|
768
|
+
/// \ingroup nanoarrow-buffer
|
|
769
|
+
struct ArrowBuffer {
|
|
770
|
+
/// \brief A pointer to the start of the buffer
|
|
771
|
+
///
|
|
772
|
+
/// If capacity_bytes is 0, this value may be NULL.
|
|
773
|
+
uint8_t* data;
|
|
774
|
+
|
|
775
|
+
/// \brief The size of the buffer in bytes
|
|
776
|
+
int64_t size_bytes;
|
|
777
|
+
|
|
778
|
+
/// \brief The capacity of the buffer in bytes
|
|
779
|
+
int64_t capacity_bytes;
|
|
780
|
+
|
|
781
|
+
/// \brief The allocator that will be used to reallocate and/or free the buffer
|
|
782
|
+
struct ArrowBufferAllocator allocator;
|
|
783
|
+
};
|
|
784
|
+
|
|
785
|
+
/// \brief An owning mutable view of a bitmap
|
|
786
|
+
/// \ingroup nanoarrow-bitmap
|
|
787
|
+
struct ArrowBitmap {
|
|
788
|
+
/// \brief An ArrowBuffer to hold the allocated memory
|
|
789
|
+
struct ArrowBuffer buffer;
|
|
790
|
+
|
|
791
|
+
/// \brief The number of bits that have been appended to the bitmap
|
|
792
|
+
int64_t size_bits;
|
|
793
|
+
};
|
|
794
|
+
|
|
795
|
+
/// \brief A description of an arrangement of buffers
|
|
796
|
+
/// \ingroup nanoarrow-utils
|
|
797
|
+
///
|
|
798
|
+
/// Contains the minimum amount of information required to
|
|
799
|
+
/// calculate the size of each buffer in an ArrowArray knowing only
|
|
800
|
+
/// the length and offset of the array.
|
|
801
|
+
struct ArrowLayout {
|
|
802
|
+
/// \brief The function of each buffer
|
|
803
|
+
enum ArrowBufferType buffer_type[NANOARROW_MAX_FIXED_BUFFERS];
|
|
804
|
+
|
|
805
|
+
/// \brief The data type of each buffer
|
|
806
|
+
enum ArrowType buffer_data_type[NANOARROW_MAX_FIXED_BUFFERS];
|
|
807
|
+
|
|
808
|
+
/// \brief The size of an element each buffer or 0 if this size is variable or unknown
|
|
809
|
+
int64_t element_size_bits[NANOARROW_MAX_FIXED_BUFFERS];
|
|
810
|
+
|
|
811
|
+
/// \brief The number of elements in the child array per element in this array for a
|
|
812
|
+
/// fixed-size list
|
|
813
|
+
int64_t child_size_elements;
|
|
814
|
+
};
|
|
815
|
+
|
|
816
|
+
/// \brief A non-owning view of an ArrowArray
|
|
817
|
+
/// \ingroup nanoarrow-array-view
|
|
818
|
+
///
|
|
819
|
+
/// This data structure provides access to the values contained within
|
|
820
|
+
/// an ArrowArray with fields provided in a more readily-extractible
|
|
821
|
+
/// form. You can re-use an ArrowArrayView for multiple ArrowArrays
|
|
822
|
+
/// with the same storage type, use it to represent a hypothetical
|
|
823
|
+
/// ArrowArray that does not exist yet, or use it to validate the buffers
|
|
824
|
+
/// of a future ArrowArray.
|
|
825
|
+
struct ArrowArrayView {
|
|
826
|
+
/// \brief The underlying ArrowArray or NULL if it has not been set or
|
|
827
|
+
/// if the buffers in this ArrowArrayView are not backed by an ArrowArray.
|
|
828
|
+
const struct ArrowArray* array;
|
|
829
|
+
|
|
830
|
+
/// \brief The number of elements from the physical start of the buffers.
|
|
831
|
+
int64_t offset;
|
|
832
|
+
|
|
833
|
+
/// \brief The number of elements in this view.
|
|
834
|
+
int64_t length;
|
|
835
|
+
|
|
836
|
+
/// \brief A cached null count or -1 to indicate that this value is unknown.
|
|
837
|
+
int64_t null_count;
|
|
838
|
+
|
|
839
|
+
/// \brief The type used to store values in this array
|
|
840
|
+
///
|
|
841
|
+
/// This type represents only the minimum required information to
|
|
842
|
+
/// extract values from the array buffers (e.g., for a Date32 array,
|
|
843
|
+
/// this value will be NANOARROW_TYPE_INT32). For dictionary-encoded
|
|
844
|
+
/// arrays, this will be the index type.
|
|
845
|
+
enum ArrowType storage_type;
|
|
846
|
+
|
|
847
|
+
/// \brief The buffer types, strides, and sizes of this Array's buffers
|
|
848
|
+
struct ArrowLayout layout;
|
|
849
|
+
|
|
850
|
+
/// \brief This Array's buffers as ArrowBufferView objects
|
|
851
|
+
struct ArrowBufferView buffer_views[NANOARROW_MAX_FIXED_BUFFERS];
|
|
852
|
+
|
|
853
|
+
/// \brief The number of children of this view
|
|
854
|
+
int64_t n_children;
|
|
855
|
+
|
|
856
|
+
/// \brief Pointers to views of this array's children
|
|
857
|
+
struct ArrowArrayView** children;
|
|
858
|
+
|
|
859
|
+
/// \brief Pointer to a view of this array's dictionary
|
|
860
|
+
struct ArrowArrayView* dictionary;
|
|
861
|
+
|
|
862
|
+
/// \brief Union type id to child index mapping
|
|
863
|
+
///
|
|
864
|
+
/// If storage_type is a union type, a 256-byte ArrowMalloc()ed buffer
|
|
865
|
+
/// such that child_index == union_type_id_map[type_id] and
|
|
866
|
+
/// type_id == union_type_id_map[128 + child_index]. This value may be
|
|
867
|
+
/// NULL in the case where child_id == type_id.
|
|
868
|
+
int8_t* union_type_id_map;
|
|
869
|
+
|
|
870
|
+
/// \brief Number of variadic buffers
|
|
871
|
+
int32_t n_variadic_buffers;
|
|
872
|
+
|
|
873
|
+
/// \brief Pointers to variadic buffers of binary/string_view arrays
|
|
874
|
+
const void** variadic_buffers;
|
|
875
|
+
|
|
876
|
+
/// \brief Size of each variadic buffer
|
|
877
|
+
int64_t* variadic_buffer_sizes;
|
|
878
|
+
};
|
|
879
|
+
|
|
880
|
+
// Used as the private data member for ArrowArrays allocated here and accessed
|
|
881
|
+
// internally within inline ArrowArray* helpers.
|
|
882
|
+
struct ArrowArrayPrivateData {
|
|
883
|
+
// Holder for the validity buffer (or first buffer for union types, which are
|
|
884
|
+
// the only type whose first buffer is not a valdiity buffer)
|
|
885
|
+
struct ArrowBitmap bitmap;
|
|
886
|
+
|
|
887
|
+
// Holder for additional buffers as required
|
|
888
|
+
struct ArrowBuffer buffers[NANOARROW_MAX_FIXED_BUFFERS - 1];
|
|
889
|
+
|
|
890
|
+
// The array of pointers to buffers. This must be updated after a sequence
|
|
891
|
+
// of appends to synchronize its values with the actual buffer addresses
|
|
892
|
+
// (which may have been reallocated during that time)
|
|
893
|
+
const void** buffer_data;
|
|
894
|
+
|
|
895
|
+
// The storage data type, or NANOARROW_TYPE_UNINITIALIZED if unknown
|
|
896
|
+
enum ArrowType storage_type;
|
|
897
|
+
|
|
898
|
+
// The buffer arrangement for the storage type
|
|
899
|
+
struct ArrowLayout layout;
|
|
900
|
+
|
|
901
|
+
// Flag to indicate if there are non-sequence union type ids.
|
|
902
|
+
// In the future this could be replaced with a type id<->child mapping
|
|
903
|
+
// to support constructing unions in append mode where type_id != child_index
|
|
904
|
+
int8_t union_type_id_is_child_index;
|
|
905
|
+
|
|
906
|
+
// Number of variadic buffers for binary view types
|
|
907
|
+
int32_t n_variadic_buffers;
|
|
908
|
+
|
|
909
|
+
// Variadic buffers for binary view types
|
|
910
|
+
struct ArrowBuffer* variadic_buffers;
|
|
911
|
+
|
|
912
|
+
// The current offset used to build list views
|
|
913
|
+
int64_t list_view_offset;
|
|
914
|
+
};
|
|
915
|
+
|
|
916
|
+
/// \brief A representation of an interval.
|
|
917
|
+
/// \ingroup nanoarrow-utils
|
|
918
|
+
struct ArrowInterval {
|
|
919
|
+
/// \brief The type of interval being used
|
|
920
|
+
enum ArrowType type;
|
|
921
|
+
/// \brief The number of months represented by the interval
|
|
922
|
+
int32_t months;
|
|
923
|
+
/// \brief The number of days represented by the interval
|
|
924
|
+
int32_t days;
|
|
925
|
+
/// \brief The number of ms represented by the interval
|
|
926
|
+
int32_t ms;
|
|
927
|
+
/// \brief The number of ns represented by the interval
|
|
928
|
+
int64_t ns;
|
|
929
|
+
};
|
|
930
|
+
|
|
931
|
+
/// \brief Zero initialize an Interval with a given unit
|
|
932
|
+
/// \ingroup nanoarrow-utils
|
|
933
|
+
static inline void ArrowIntervalInit(struct ArrowInterval* interval,
|
|
934
|
+
enum ArrowType type) {
|
|
935
|
+
memset(interval, 0, sizeof(struct ArrowInterval));
|
|
936
|
+
interval->type = type;
|
|
937
|
+
}
|
|
938
|
+
|
|
939
|
+
/// \brief A representation of a fixed-precision decimal number
|
|
940
|
+
/// \ingroup nanoarrow-utils
|
|
941
|
+
///
|
|
942
|
+
/// This structure should be initialized with ArrowDecimalInit() once and
|
|
943
|
+
/// values set using ArrowDecimalSetInt(), ArrowDecimalSetBytes128(),
|
|
944
|
+
/// or ArrowDecimalSetBytes256().
|
|
945
|
+
struct ArrowDecimal {
|
|
946
|
+
/// \brief An array of 64-bit integers of n_words length defined in native-endian order.
|
|
947
|
+
/// For a 32-bit decimal value, index 0 will be a 32-bit integer value.
|
|
948
|
+
uint64_t words[4];
|
|
949
|
+
|
|
950
|
+
/// \brief The number of significant digits this decimal number can represent
|
|
951
|
+
int32_t precision;
|
|
952
|
+
|
|
953
|
+
/// \brief The number of digits after the decimal point. This can be negative.
|
|
954
|
+
int32_t scale;
|
|
955
|
+
|
|
956
|
+
/// \brief The number of 64-bit words in the words array. For the special case of a
|
|
957
|
+
/// 32-bit decimal value, this will be 0.
|
|
958
|
+
int n_words;
|
|
959
|
+
|
|
960
|
+
/// \brief Cached value used by the implementation
|
|
961
|
+
int high_word_index;
|
|
962
|
+
|
|
963
|
+
/// \brief Cached value used by the implementation
|
|
964
|
+
int low_word_index;
|
|
965
|
+
};
|
|
966
|
+
|
|
967
|
+
/// \brief Initialize a decimal with a given set of type parameters
|
|
968
|
+
/// \ingroup nanoarrow-utils
|
|
969
|
+
static inline void ArrowDecimalInit(struct ArrowDecimal* decimal, int32_t bitwidth,
|
|
970
|
+
int32_t precision, int32_t scale) {
|
|
971
|
+
memset(decimal->words, 0, sizeof(decimal->words));
|
|
972
|
+
decimal->precision = precision;
|
|
973
|
+
decimal->scale = scale;
|
|
974
|
+
// n_words will be 0 for bitwidth == 32
|
|
975
|
+
decimal->n_words = (int)(bitwidth / 8 / sizeof(uint64_t));
|
|
976
|
+
|
|
977
|
+
if (_ArrowIsLittleEndian()) {
|
|
978
|
+
decimal->low_word_index = 0;
|
|
979
|
+
decimal->high_word_index = decimal->n_words > 0 ? decimal->n_words - 1 : 0;
|
|
980
|
+
} else {
|
|
981
|
+
decimal->low_word_index = decimal->n_words > 0 ? decimal->n_words - 1 : 0;
|
|
982
|
+
decimal->high_word_index = 0;
|
|
983
|
+
}
|
|
984
|
+
}
|
|
985
|
+
|
|
986
|
+
/// \brief Get a signed integer value of a sufficiently small ArrowDecimal
|
|
987
|
+
///
|
|
988
|
+
/// This does not check if the decimal's precision sufficiently small to fit
|
|
989
|
+
/// within the signed 64-bit integer range (A precision less than or equal
|
|
990
|
+
/// to 18 is sufficiently small).
|
|
991
|
+
static inline int64_t ArrowDecimalGetIntUnsafe(const struct ArrowDecimal* decimal) {
|
|
992
|
+
if (decimal->n_words == 0) {
|
|
993
|
+
int32_t value;
|
|
994
|
+
memcpy(&value, decimal->words, sizeof(int32_t));
|
|
995
|
+
return value;
|
|
996
|
+
}
|
|
997
|
+
|
|
998
|
+
return (int64_t)decimal->words[decimal->low_word_index];
|
|
999
|
+
}
|
|
1000
|
+
|
|
1001
|
+
/// \brief Copy the bytes of this decimal into a sufficiently large buffer
|
|
1002
|
+
/// \ingroup nanoarrow-utils
|
|
1003
|
+
static inline void ArrowDecimalGetBytes(const struct ArrowDecimal* decimal,
|
|
1004
|
+
uint8_t* out) {
|
|
1005
|
+
if (decimal->n_words == 0) {
|
|
1006
|
+
memcpy(out, decimal->words, sizeof(int32_t));
|
|
1007
|
+
} else {
|
|
1008
|
+
memcpy(out, decimal->words, decimal->n_words * sizeof(uint64_t));
|
|
1009
|
+
}
|
|
1010
|
+
}
|
|
1011
|
+
|
|
1012
|
+
/// \brief Returns 1 if the value represented by decimal is >= 0 or -1 otherwise
|
|
1013
|
+
/// \ingroup nanoarrow-utils
|
|
1014
|
+
static inline int64_t ArrowDecimalSign(const struct ArrowDecimal* decimal) {
|
|
1015
|
+
if (decimal->n_words == 0) {
|
|
1016
|
+
return ArrowDecimalGetIntUnsafe(decimal) >= 0 ? 1 : -1;
|
|
1017
|
+
} else {
|
|
1018
|
+
return 1 | ((int64_t)(decimal->words[decimal->high_word_index]) >> 63);
|
|
1019
|
+
}
|
|
1020
|
+
}
|
|
1021
|
+
|
|
1022
|
+
/// \brief Sets the integer value of this decimal
|
|
1023
|
+
/// \ingroup nanoarrow-utils
|
|
1024
|
+
static inline void ArrowDecimalSetInt(struct ArrowDecimal* decimal, int64_t value) {
|
|
1025
|
+
if (decimal->n_words == 0) {
|
|
1026
|
+
int32_t value32 = (int32_t)value;
|
|
1027
|
+
memcpy(decimal->words, &value32, sizeof(int32_t));
|
|
1028
|
+
return;
|
|
1029
|
+
}
|
|
1030
|
+
|
|
1031
|
+
if (value < 0) {
|
|
1032
|
+
memset(decimal->words, 0xff, decimal->n_words * sizeof(uint64_t));
|
|
1033
|
+
} else {
|
|
1034
|
+
memset(decimal->words, 0, decimal->n_words * sizeof(uint64_t));
|
|
1035
|
+
}
|
|
1036
|
+
|
|
1037
|
+
decimal->words[decimal->low_word_index] = value;
|
|
1038
|
+
}
|
|
1039
|
+
|
|
1040
|
+
/// \brief Negate the value of this decimal in place
|
|
1041
|
+
/// \ingroup nanoarrow-utils
|
|
1042
|
+
static inline void ArrowDecimalNegate(struct ArrowDecimal* decimal) {
|
|
1043
|
+
if (decimal->n_words == 0) {
|
|
1044
|
+
int32_t value;
|
|
1045
|
+
memcpy(&value, decimal->words, sizeof(int32_t));
|
|
1046
|
+
value = -value;
|
|
1047
|
+
memcpy(decimal->words, &value, sizeof(int32_t));
|
|
1048
|
+
return;
|
|
1049
|
+
}
|
|
1050
|
+
|
|
1051
|
+
uint64_t carry = 1;
|
|
1052
|
+
|
|
1053
|
+
if (decimal->low_word_index == 0) {
|
|
1054
|
+
for (int i = 0; i < decimal->n_words; i++) {
|
|
1055
|
+
uint64_t elem = decimal->words[i];
|
|
1056
|
+
elem = ~elem + carry;
|
|
1057
|
+
carry &= (elem == 0);
|
|
1058
|
+
decimal->words[i] = elem;
|
|
1059
|
+
}
|
|
1060
|
+
} else {
|
|
1061
|
+
for (int i = decimal->low_word_index; i >= 0; i--) {
|
|
1062
|
+
uint64_t elem = decimal->words[i];
|
|
1063
|
+
elem = ~elem + carry;
|
|
1064
|
+
carry &= (elem == 0);
|
|
1065
|
+
decimal->words[i] = elem;
|
|
1066
|
+
}
|
|
1067
|
+
}
|
|
1068
|
+
}
|
|
1069
|
+
|
|
1070
|
+
/// \brief Copy bytes from a buffer into this decimal
|
|
1071
|
+
/// \ingroup nanoarrow-utils
|
|
1072
|
+
static inline void ArrowDecimalSetBytes(struct ArrowDecimal* decimal,
|
|
1073
|
+
const uint8_t* value) {
|
|
1074
|
+
if (decimal->n_words == 0) {
|
|
1075
|
+
memcpy(decimal->words, value, sizeof(int32_t));
|
|
1076
|
+
} else {
|
|
1077
|
+
memcpy(decimal->words, value, decimal->n_words * sizeof(uint64_t));
|
|
1078
|
+
}
|
|
1079
|
+
}
|
|
1080
|
+
|
|
1081
|
+
#ifdef __cplusplus
|
|
1082
|
+
}
|
|
1083
|
+
#endif
|
|
1084
|
+
|
|
1085
|
+
#endif
|
|
1086
|
+
// Licensed to the Apache Software Foundation (ASF) under one
|
|
1087
|
+
// or more contributor license agreements. See the NOTICE file
|
|
1088
|
+
// distributed with this work for additional information
|
|
1089
|
+
// regarding copyright ownership. The ASF licenses this file
|
|
1090
|
+
// to you under the Apache License, Version 2.0 (the
|
|
1091
|
+
// "License"); you may not use this file except in compliance
|
|
1092
|
+
// with the License. You may obtain a copy of the License at
|
|
1093
|
+
//
|
|
1094
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
1095
|
+
//
|
|
1096
|
+
// Unless required by applicable law or agreed to in writing,
|
|
1097
|
+
// software distributed under the License is distributed on an
|
|
1098
|
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
1099
|
+
// KIND, either express or implied. See the License for the
|
|
1100
|
+
// specific language governing permissions and limitations
|
|
1101
|
+
// under the License.
|
|
1102
|
+
|
|
1103
|
+
#ifndef NANOARROW_H_INCLUDED
|
|
1104
|
+
#define NANOARROW_H_INCLUDED
|
|
1105
|
+
|
|
1106
|
+
#include <stddef.h>
|
|
1107
|
+
#include <stdint.h>
|
|
1108
|
+
#include <stdlib.h>
|
|
1109
|
+
|
|
1110
|
+
|
|
1111
|
+
|
|
1112
|
+
// If using CMake, optionally pass -DNANOARROW_NAMESPACE=MyNamespace which will set this
|
|
1113
|
+
// define in nanoarrow_config.h. If not, you can optionally #define NANOARROW_NAMESPACE
|
|
1114
|
+
// MyNamespace here.
|
|
1115
|
+
|
|
1116
|
+
// This section remaps the non-prefixed symbols to the prefixed symbols so that
|
|
1117
|
+
// code written against this build can be used independent of the value of
|
|
1118
|
+
// NANOARROW_NAMESPACE.
|
|
1119
|
+
#ifdef NANOARROW_NAMESPACE
|
|
1120
|
+
#define NANOARROW_CAT(A, B) A##B
|
|
1121
|
+
#define NANOARROW_SYMBOL(A, B) NANOARROW_CAT(A, B)
|
|
1122
|
+
|
|
1123
|
+
#define ArrowNanoarrowVersion NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowNanoarrowVersion)
|
|
1124
|
+
#define ArrowNanoarrowVersionInt \
|
|
1125
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowNanoarrowVersionInt)
|
|
1126
|
+
#define ArrowMalloc NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMalloc)
|
|
1127
|
+
#define ArrowRealloc NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowRealloc)
|
|
1128
|
+
#define ArrowFree NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowFree)
|
|
1129
|
+
#define ArrowBufferAllocatorDefault \
|
|
1130
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowBufferAllocatorDefault)
|
|
1131
|
+
#define ArrowBufferDeallocator \
|
|
1132
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowBufferDeallocator)
|
|
1133
|
+
#define ArrowErrorSet NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowErrorSet)
|
|
1134
|
+
#define ArrowLayoutInit NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowLayoutInit)
|
|
1135
|
+
#define ArrowDecimalSetDigits NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowDecimalSetDigits)
|
|
1136
|
+
#define ArrowDecimalAppendDigitsToBuffer \
|
|
1137
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowDecimalAppendDigitsToBuffer)
|
|
1138
|
+
#define ArrowDecimalAppendStringToBuffer \
|
|
1139
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowDecimalAppendStringToBuffer)
|
|
1140
|
+
#define ArrowSchemaInit NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaInit)
|
|
1141
|
+
#define ArrowSchemaInitFromType \
|
|
1142
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaInitFromType)
|
|
1143
|
+
#define ArrowSchemaSetType NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetType)
|
|
1144
|
+
#define ArrowSchemaSetTypeStruct \
|
|
1145
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetTypeStruct)
|
|
1146
|
+
#define ArrowSchemaSetTypeFixedSize \
|
|
1147
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetTypeFixedSize)
|
|
1148
|
+
#define ArrowSchemaSetTypeDecimal \
|
|
1149
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetTypeDecimal)
|
|
1150
|
+
#define ArrowSchemaSetTypeRunEndEncoded \
|
|
1151
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetTypeRunEndEncoded)
|
|
1152
|
+
#define ArrowSchemaSetTypeDateTime \
|
|
1153
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetTypeDateTime)
|
|
1154
|
+
#define ArrowSchemaSetTypeUnion \
|
|
1155
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetTypeUnion)
|
|
1156
|
+
#define ArrowSchemaDeepCopy NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaDeepCopy)
|
|
1157
|
+
#define ArrowSchemaSetFormat NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetFormat)
|
|
1158
|
+
#define ArrowSchemaSetName NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetName)
|
|
1159
|
+
#define ArrowSchemaSetMetadata \
|
|
1160
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetMetadata)
|
|
1161
|
+
#define ArrowSchemaAllocateChildren \
|
|
1162
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaAllocateChildren)
|
|
1163
|
+
#define ArrowSchemaAllocateDictionary \
|
|
1164
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaAllocateDictionary)
|
|
1165
|
+
#define ArrowMetadataReaderInit \
|
|
1166
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataReaderInit)
|
|
1167
|
+
#define ArrowMetadataReaderRead \
|
|
1168
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataReaderRead)
|
|
1169
|
+
#define ArrowMetadataSizeOf NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataSizeOf)
|
|
1170
|
+
#define ArrowMetadataHasKey NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataHasKey)
|
|
1171
|
+
#define ArrowMetadataGetValue NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataGetValue)
|
|
1172
|
+
#define ArrowMetadataBuilderInit \
|
|
1173
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataBuilderInit)
|
|
1174
|
+
#define ArrowMetadataBuilderAppend \
|
|
1175
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataBuilderAppend)
|
|
1176
|
+
#define ArrowMetadataBuilderSet \
|
|
1177
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataBuilderSet)
|
|
1178
|
+
#define ArrowMetadataBuilderRemove \
|
|
1179
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataBuilderRemove)
|
|
1180
|
+
#define ArrowSchemaViewInit NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaViewInit)
|
|
1181
|
+
#define ArrowSchemaToString NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaToString)
|
|
1182
|
+
#define ArrowArrayInitFromType \
|
|
1183
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayInitFromType)
|
|
1184
|
+
#define ArrowArrayInitFromSchema \
|
|
1185
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayInitFromSchema)
|
|
1186
|
+
#define ArrowArrayInitFromArrayView \
|
|
1187
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayInitFromArrayView)
|
|
1188
|
+
#define ArrowArrayInitFromArrayView \
|
|
1189
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayInitFromArrayView)
|
|
1190
|
+
#define ArrowArrayAllocateChildren \
|
|
1191
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayAllocateChildren)
|
|
1192
|
+
#define ArrowArrayAllocateDictionary \
|
|
1193
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayAllocateDictionary)
|
|
1194
|
+
#define ArrowArraySetValidityBitmap \
|
|
1195
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArraySetValidityBitmap)
|
|
1196
|
+
#define ArrowArraySetBuffer NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArraySetBuffer)
|
|
1197
|
+
#define ArrowArrayReserve NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayReserve)
|
|
1198
|
+
#define ArrowArrayFinishBuilding \
|
|
1199
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayFinishBuilding)
|
|
1200
|
+
#define ArrowArrayFinishBuildingDefault \
|
|
1201
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayFinishBuildingDefault)
|
|
1202
|
+
#define ArrowArrayViewInitFromType \
|
|
1203
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayViewInitFromType)
|
|
1204
|
+
#define ArrowArrayViewInitFromSchema \
|
|
1205
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayViewInitFromSchema)
|
|
1206
|
+
#define ArrowArrayViewAllocateChildren \
|
|
1207
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayViewAllocateChildren)
|
|
1208
|
+
#define ArrowArrayViewAllocateDictionary \
|
|
1209
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayViewAllocateDictionary)
|
|
1210
|
+
#define ArrowArrayViewSetLength \
|
|
1211
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayViewSetLength)
|
|
1212
|
+
#define ArrowArrayViewSetArray \
|
|
1213
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayViewSetArray)
|
|
1214
|
+
#define ArrowArrayViewSetArrayMinimal \
|
|
1215
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayViewSetArrayMinimal)
|
|
1216
|
+
#define ArrowArrayViewValidate \
|
|
1217
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayViewValidate)
|
|
1218
|
+
#define ArrowArrayViewCompare NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayViewCompare)
|
|
1219
|
+
#define ArrowArrayViewReset NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayViewReset)
|
|
1220
|
+
#define ArrowBasicArrayStreamInit \
|
|
1221
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowBasicArrayStreamInit)
|
|
1222
|
+
#define ArrowBasicArrayStreamSetArray \
|
|
1223
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowBasicArrayStreamSetArray)
|
|
1224
|
+
#define ArrowBasicArrayStreamValidate \
|
|
1225
|
+
NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowBasicArrayStreamValidate)
|
|
1226
|
+
|
|
1227
|
+
#endif
|
|
1228
|
+
|
|
1229
|
+
#if (defined _WIN32 || defined __CYGWIN__) && defined(NANOARROW_BUILD_DLL)
|
|
1230
|
+
#if defined(NANOARROW_EXPORT_DLL)
|
|
1231
|
+
#define NANOARROW_DLL __declspec(dllexport)
|
|
1232
|
+
#else
|
|
1233
|
+
#define NANOARROW_DLL __declspec(dllimport)
|
|
1234
|
+
#endif // defined(NANOARROW_EXPORT_DLL)
|
|
1235
|
+
#elif !defined(NANOARROW_DLL)
|
|
1236
|
+
#if defined(__GNUC__) && __GNUC__ >= 4
|
|
1237
|
+
#define NANOARROW_DLL __attribute__((visibility("default")))
|
|
1238
|
+
#else
|
|
1239
|
+
#define NANOARROW_DLL
|
|
1240
|
+
#endif // __GNUC__ >= 4
|
|
1241
|
+
#endif
|
|
1242
|
+
|
|
1243
|
+
#ifdef __cplusplus
|
|
1244
|
+
extern "C" {
|
|
1245
|
+
#endif
|
|
1246
|
+
|
|
1247
|
+
/// \defgroup nanoarrow Nanoarrow C library
|
|
1248
|
+
///
|
|
1249
|
+
/// Except where noted, objects are not thread-safe and clients should
|
|
1250
|
+
/// take care to serialize accesses to methods.
|
|
1251
|
+
///
|
|
1252
|
+
/// Because this library is intended to be vendored, it provides full type
|
|
1253
|
+
/// definitions and encourages clients to stack or statically allocate
|
|
1254
|
+
/// where convenient.
|
|
1255
|
+
|
|
1256
|
+
/// \defgroup nanoarrow-malloc Memory management
|
|
1257
|
+
///
|
|
1258
|
+
/// Non-buffer members of a struct ArrowSchema and struct ArrowArray
|
|
1259
|
+
/// must be allocated using ArrowMalloc() or ArrowRealloc() and freed
|
|
1260
|
+
/// using ArrowFree() for schemas and arrays allocated here. Buffer members
|
|
1261
|
+
/// are allocated using an ArrowBufferAllocator.
|
|
1262
|
+
///
|
|
1263
|
+
/// @{
|
|
1264
|
+
|
|
1265
|
+
/// \brief Allocate like malloc()
|
|
1266
|
+
NANOARROW_DLL void* ArrowMalloc(int64_t size);
|
|
1267
|
+
|
|
1268
|
+
/// \brief Reallocate like realloc()
|
|
1269
|
+
NANOARROW_DLL void* ArrowRealloc(void* ptr, int64_t size);
|
|
1270
|
+
|
|
1271
|
+
/// \brief Free a pointer allocated using ArrowMalloc() or ArrowRealloc().
|
|
1272
|
+
NANOARROW_DLL void ArrowFree(void* ptr);
|
|
1273
|
+
|
|
1274
|
+
/// \brief Return the default allocator
|
|
1275
|
+
///
|
|
1276
|
+
/// The default allocator uses ArrowMalloc(), ArrowRealloc(), and
|
|
1277
|
+
/// ArrowFree().
|
|
1278
|
+
NANOARROW_DLL struct ArrowBufferAllocator ArrowBufferAllocatorDefault(void);
|
|
1279
|
+
|
|
1280
|
+
/// \brief Create a custom deallocator
|
|
1281
|
+
///
|
|
1282
|
+
/// Creates a buffer allocator with only a free method that can be used to
|
|
1283
|
+
/// attach a custom deallocator to an ArrowBuffer. This may be used to
|
|
1284
|
+
/// avoid copying an existing buffer that was not allocated using the
|
|
1285
|
+
/// infrastructure provided here (e.g., by an R or Python object).
|
|
1286
|
+
NANOARROW_DLL struct ArrowBufferAllocator ArrowBufferDeallocator(
|
|
1287
|
+
ArrowBufferDeallocatorCallback, void* private_data);
|
|
1288
|
+
|
|
1289
|
+
/// @}
|
|
1290
|
+
|
|
1291
|
+
/// \brief Move the contents of an src ArrowSchema into dst and set src->release to NULL
|
|
1292
|
+
/// \ingroup nanoarrow-arrow-cdata
|
|
1293
|
+
static inline void ArrowSchemaMove(struct ArrowSchema* src, struct ArrowSchema* dst);
|
|
1294
|
+
|
|
1295
|
+
/// \brief Call the release callback of an ArrowSchema
|
|
1296
|
+
/// \ingroup nanoarrow-arrow-cdata
|
|
1297
|
+
static inline void ArrowSchemaRelease(struct ArrowSchema* schema);
|
|
1298
|
+
|
|
1299
|
+
/// \brief Move the contents of an src ArrowArray into dst and set src->release to NULL
|
|
1300
|
+
/// \ingroup nanoarrow-arrow-cdata
|
|
1301
|
+
static inline void ArrowArrayMove(struct ArrowArray* src, struct ArrowArray* dst);
|
|
1302
|
+
|
|
1303
|
+
/// \brief Call the release callback of an ArrowArray
|
|
1304
|
+
static inline void ArrowArrayRelease(struct ArrowArray* array);
|
|
1305
|
+
|
|
1306
|
+
/// \brief Move the contents of an src ArrowArrayStream into dst and set src->release to
|
|
1307
|
+
/// NULL \ingroup nanoarrow-arrow-cdata
|
|
1308
|
+
static inline void ArrowArrayStreamMove(struct ArrowArrayStream* src,
|
|
1309
|
+
struct ArrowArrayStream* dst);
|
|
1310
|
+
|
|
1311
|
+
/// \brief Call the get_schema callback of an ArrowArrayStream
|
|
1312
|
+
/// \ingroup nanoarrow-arrow-cdata
|
|
1313
|
+
///
|
|
1314
|
+
/// Unlike the get_schema callback, this wrapper checks the return code
|
|
1315
|
+
/// and propagates the error reported by get_last_error into error. This
|
|
1316
|
+
/// makes it significantly less verbose to iterate over array streams
|
|
1317
|
+
/// using NANOARROW_RETURN_NOT_OK()-style error handling.
|
|
1318
|
+
static inline ArrowErrorCode ArrowArrayStreamGetSchema(
|
|
1319
|
+
struct ArrowArrayStream* array_stream, struct ArrowSchema* out,
|
|
1320
|
+
struct ArrowError* error);
|
|
1321
|
+
|
|
1322
|
+
/// \brief Call the get_next callback of an ArrowArrayStream
|
|
1323
|
+
/// \ingroup nanoarrow-arrow-cdata
|
|
1324
|
+
///
|
|
1325
|
+
/// Unlike the get_next callback, this wrapper checks the return code
|
|
1326
|
+
/// and propagates the error reported by get_last_error into error. This
|
|
1327
|
+
/// makes it significantly less verbose to iterate over array streams
|
|
1328
|
+
/// using NANOARROW_RETURN_NOT_OK()-style error handling.
|
|
1329
|
+
static inline ArrowErrorCode ArrowArrayStreamGetNext(
|
|
1330
|
+
struct ArrowArrayStream* array_stream, struct ArrowArray* out,
|
|
1331
|
+
struct ArrowError* error);
|
|
1332
|
+
|
|
1333
|
+
/// \brief Call the get_last_error callback of an ArrowArrayStream
|
|
1334
|
+
/// \ingroup nanoarrow-arrow-cdata
|
|
1335
|
+
///
|
|
1336
|
+
/// Unlike the get_last_error callback, this function never returns NULL (i.e.,
|
|
1337
|
+
/// its result is safe to use in printf-style error formatters). Null values
|
|
1338
|
+
/// from the original callback are reported as
|
|
1339
|
+
/// "<get_last_error() returned NULL>".
|
|
1340
|
+
static inline const char* ArrowArrayStreamGetLastError(
|
|
1341
|
+
struct ArrowArrayStream* array_stream);
|
|
1342
|
+
|
|
1343
|
+
/// \brief Call the release callback of an ArrowArrayStream
|
|
1344
|
+
static inline void ArrowArrayStreamRelease(struct ArrowArrayStream* array_stream);
|
|
1345
|
+
|
|
1346
|
+
/// \defgroup nanoarrow-errors Error handling
|
|
1347
|
+
///
|
|
1348
|
+
/// Functions generally return an errno-compatible error code; functions that
|
|
1349
|
+
/// need to communicate more verbose error information accept a pointer
|
|
1350
|
+
/// to an ArrowError. This can be stack or statically allocated. The
|
|
1351
|
+
/// content of the message is undefined unless an error code has been
|
|
1352
|
+
/// returned. If a nanoarrow function is passed a non-null ArrowError pointer, the
|
|
1353
|
+
/// ArrowError pointed to by the argument will be propagated with a
|
|
1354
|
+
/// null-terminated error message. It is safe to pass a NULL ArrowError anywhere
|
|
1355
|
+
/// in the nanoarrow API.
|
|
1356
|
+
///
|
|
1357
|
+
/// Except where documented, it is generally not safe to continue after a
|
|
1358
|
+
/// function has returned a non-zero ArrowErrorCode. The NANOARROW_RETURN_NOT_OK and
|
|
1359
|
+
/// NANOARROW_ASSERT_OK macros are provided to help propagate errors. C++ clients can use
|
|
1360
|
+
/// the helpers provided in the nanoarrow.hpp header to facilitate using C++ idioms
|
|
1361
|
+
/// for memory management and error propgagtion.
|
|
1362
|
+
///
|
|
1363
|
+
/// @{
|
|
1364
|
+
|
|
1365
|
+
/// \brief Set the contents of an error using printf syntax.
|
|
1366
|
+
///
|
|
1367
|
+
/// If error is NULL, this function does nothing and returns NANOARROW_OK.
|
|
1368
|
+
NANOARROW_DLL NANOARROW_CHECK_PRINTF_ATTRIBUTE int ArrowErrorSet(struct ArrowError* error,
|
|
1369
|
+
const char* fmt, ...);
|
|
1370
|
+
|
|
1371
|
+
/// @}
|
|
1372
|
+
|
|
1373
|
+
/// \defgroup nanoarrow-utils Utility data structures
|
|
1374
|
+
///
|
|
1375
|
+
/// @{
|
|
1376
|
+
|
|
1377
|
+
/// \brief Return a version string in the form "major.minor.patch"
|
|
1378
|
+
NANOARROW_DLL const char* ArrowNanoarrowVersion(void);
|
|
1379
|
+
|
|
1380
|
+
/// \brief Return an integer that can be used to compare versions sequentially
|
|
1381
|
+
NANOARROW_DLL int ArrowNanoarrowVersionInt(void);
|
|
1382
|
+
|
|
1383
|
+
/// \brief Initialize a description of buffer arrangements from a storage type
|
|
1384
|
+
NANOARROW_DLL void ArrowLayoutInit(struct ArrowLayout* layout,
|
|
1385
|
+
enum ArrowType storage_type);
|
|
1386
|
+
|
|
1387
|
+
/// \brief Create a string view from a null-terminated string
|
|
1388
|
+
static inline struct ArrowStringView ArrowCharView(const char* value);
|
|
1389
|
+
|
|
1390
|
+
/// \brief Sets the integer value of an ArrowDecimal from a string
|
|
1391
|
+
NANOARROW_DLL ArrowErrorCode ArrowDecimalSetDigits(struct ArrowDecimal* decimal,
|
|
1392
|
+
struct ArrowStringView value);
|
|
1393
|
+
|
|
1394
|
+
/// \brief Get the integer value of an ArrowDecimal as string
|
|
1395
|
+
NANOARROW_DLL ArrowErrorCode ArrowDecimalAppendDigitsToBuffer(
|
|
1396
|
+
const struct ArrowDecimal* decimal, struct ArrowBuffer* buffer);
|
|
1397
|
+
|
|
1398
|
+
/// \brief Get the decimal value of an ArrowDecimal as a string
|
|
1399
|
+
NANOARROW_DLL ArrowErrorCode ArrowDecimalAppendStringToBuffer(
|
|
1400
|
+
const struct ArrowDecimal* decimal, struct ArrowBuffer* buffer);
|
|
1401
|
+
|
|
1402
|
+
/// \brief Get the half float value of a float
|
|
1403
|
+
static inline uint16_t ArrowFloatToHalfFloat(float value);
|
|
1404
|
+
|
|
1405
|
+
/// \brief Get the float value of a half float
|
|
1406
|
+
static inline float ArrowHalfFloatToFloat(uint16_t value);
|
|
1407
|
+
|
|
1408
|
+
/// \brief Resolve a chunk index from increasing int64_t offsets
|
|
1409
|
+
///
|
|
1410
|
+
/// Given a buffer of increasing int64_t offsets that begin with 0 (e.g., offset buffer
|
|
1411
|
+
/// of a large type, run ends of a chunked array implementation), resolve a value v
|
|
1412
|
+
/// where lo <= v < hi such that offsets[v] <= index < offsets[v + 1].
|
|
1413
|
+
static inline int64_t ArrowResolveChunk64(int64_t index, const int64_t* offsets,
|
|
1414
|
+
int64_t lo, int64_t hi);
|
|
1415
|
+
|
|
1416
|
+
/// @}
|
|
1417
|
+
|
|
1418
|
+
/// \defgroup nanoarrow-schema Creating schemas
|
|
1419
|
+
///
|
|
1420
|
+
/// These functions allocate, copy, and destroy ArrowSchema structures
|
|
1421
|
+
///
|
|
1422
|
+
/// @{
|
|
1423
|
+
|
|
1424
|
+
/// \brief Initialize an ArrowSchema
|
|
1425
|
+
///
|
|
1426
|
+
/// Initializes the fields and release callback of schema_out. Caller
|
|
1427
|
+
/// is responsible for calling the schema->release callback if
|
|
1428
|
+
/// NANOARROW_OK is returned.
|
|
1429
|
+
NANOARROW_DLL void ArrowSchemaInit(struct ArrowSchema* schema);
|
|
1430
|
+
|
|
1431
|
+
/// \brief Initialize an ArrowSchema from an ArrowType
|
|
1432
|
+
///
|
|
1433
|
+
/// A convenience constructor for that calls ArrowSchemaInit() and
|
|
1434
|
+
/// ArrowSchemaSetType() for the common case of constructing an
|
|
1435
|
+
/// unparameterized type. The caller is responsible for calling the schema->release
|
|
1436
|
+
/// callback if NANOARROW_OK is returned.
|
|
1437
|
+
NANOARROW_DLL ArrowErrorCode ArrowSchemaInitFromType(struct ArrowSchema* schema,
|
|
1438
|
+
enum ArrowType type);
|
|
1439
|
+
|
|
1440
|
+
/// \brief Get a human-readable summary of a Schema
|
|
1441
|
+
///
|
|
1442
|
+
/// Writes a summary of an ArrowSchema to out (up to n - 1 characters)
|
|
1443
|
+
/// and returns the number of characters required for the output if
|
|
1444
|
+
/// n were sufficiently large. If recursive is non-zero, the result will
|
|
1445
|
+
/// also include children.
|
|
1446
|
+
NANOARROW_DLL int64_t ArrowSchemaToString(const struct ArrowSchema* schema, char* out,
|
|
1447
|
+
int64_t n, char recursive);
|
|
1448
|
+
|
|
1449
|
+
/// \brief Set the format field of a schema from an ArrowType
|
|
1450
|
+
///
|
|
1451
|
+
/// Initializes the fields and release callback of schema_out. For
|
|
1452
|
+
/// NANOARROW_TYPE_LIST, NANOARROW_TYPE_LARGE_LIST, and
|
|
1453
|
+
/// NANOARROW_TYPE_MAP, the appropriate number of children are
|
|
1454
|
+
/// allocated, initialized, and named; however, the caller must
|
|
1455
|
+
/// ArrowSchemaSetType() on the preinitialized children. Schema must have been initialized
|
|
1456
|
+
/// using ArrowSchemaInit() or ArrowSchemaDeepCopy().
|
|
1457
|
+
NANOARROW_DLL ArrowErrorCode ArrowSchemaSetType(struct ArrowSchema* schema,
|
|
1458
|
+
enum ArrowType type);
|
|
1459
|
+
|
|
1460
|
+
/// \brief Set the format field and initialize children of a struct schema
|
|
1461
|
+
///
|
|
1462
|
+
/// The specified number of children are initialized; however, the caller is responsible
|
|
1463
|
+
/// for calling ArrowSchemaSetType() and ArrowSchemaSetName() on each child.
|
|
1464
|
+
/// Schema must have been initialized using ArrowSchemaInit() or ArrowSchemaDeepCopy().
|
|
1465
|
+
NANOARROW_DLL ArrowErrorCode ArrowSchemaSetTypeStruct(struct ArrowSchema* schema,
|
|
1466
|
+
int64_t n_children);
|
|
1467
|
+
|
|
1468
|
+
/// \brief Set the format field of a fixed-size schema
|
|
1469
|
+
///
|
|
1470
|
+
/// Returns EINVAL for fixed_size <= 0 or for type that is not
|
|
1471
|
+
/// NANOARROW_TYPE_FIXED_SIZE_BINARY or NANOARROW_TYPE_FIXED_SIZE_LIST.
|
|
1472
|
+
/// For NANOARROW_TYPE_FIXED_SIZE_LIST, the appropriate number of children are
|
|
1473
|
+
/// allocated, initialized, and named; however, the caller must
|
|
1474
|
+
/// ArrowSchemaSetType() the first child. Schema must have been initialized using
|
|
1475
|
+
/// ArrowSchemaInit() or ArrowSchemaDeepCopy().
|
|
1476
|
+
NANOARROW_DLL ArrowErrorCode ArrowSchemaSetTypeFixedSize(struct ArrowSchema* schema,
|
|
1477
|
+
enum ArrowType type,
|
|
1478
|
+
int32_t fixed_size);
|
|
1479
|
+
|
|
1480
|
+
/// \brief Set the format field of a decimal schema
|
|
1481
|
+
///
|
|
1482
|
+
/// Returns EINVAL for scale <= 0 or for type that is not
|
|
1483
|
+
/// NANOARROW_TYPE_DECIMAL32, NANOARROW_TYPE_DECIMAL64, NANOARROW_TYPE_DECIMAL128 or
|
|
1484
|
+
/// NANOARROW_TYPE_DECIMAL256. Schema must have been initialized using
|
|
1485
|
+
/// ArrowSchemaInit() or ArrowSchemaDeepCopy().
|
|
1486
|
+
NANOARROW_DLL ArrowErrorCode ArrowSchemaSetTypeDecimal(struct ArrowSchema* schema,
|
|
1487
|
+
enum ArrowType type,
|
|
1488
|
+
int32_t decimal_precision,
|
|
1489
|
+
int32_t decimal_scale);
|
|
1490
|
+
|
|
1491
|
+
/// \brief Set the format field of a run-end encoded schema
|
|
1492
|
+
///
|
|
1493
|
+
/// Returns EINVAL for run_end_type that is not
|
|
1494
|
+
/// NANOARROW_TYPE_INT16, NANOARROW_TYPE_INT32 or NANOARROW_TYPE_INT64.
|
|
1495
|
+
/// Schema must have been initialized using ArrowSchemaInit() or ArrowSchemaDeepCopy().
|
|
1496
|
+
/// The caller must call `ArrowSchemaSetTypeXXX(schema->children[1])` to
|
|
1497
|
+
/// set the value type. Note that when building arrays using the `ArrowArrayAppendXXX()`
|
|
1498
|
+
/// functions, the run-end encoded array's logical length must be updated manually.
|
|
1499
|
+
NANOARROW_DLL ArrowErrorCode ArrowSchemaSetTypeRunEndEncoded(struct ArrowSchema* schema,
|
|
1500
|
+
enum ArrowType run_end_type);
|
|
1501
|
+
|
|
1502
|
+
/// \brief Set the format field of a time, timestamp, or duration schema
|
|
1503
|
+
///
|
|
1504
|
+
/// Returns EINVAL for type that is not
|
|
1505
|
+
/// NANOARROW_TYPE_TIME32, NANOARROW_TYPE_TIME64,
|
|
1506
|
+
/// NANOARROW_TYPE_TIMESTAMP, or NANOARROW_TYPE_DURATION. The
|
|
1507
|
+
/// timezone parameter must be NULL for a non-timestamp type. Schema must have been
|
|
1508
|
+
/// initialized using ArrowSchemaInit() or ArrowSchemaDeepCopy().
|
|
1509
|
+
NANOARROW_DLL ArrowErrorCode ArrowSchemaSetTypeDateTime(struct ArrowSchema* schema,
|
|
1510
|
+
enum ArrowType type,
|
|
1511
|
+
enum ArrowTimeUnit time_unit,
|
|
1512
|
+
const char* timezone);
|
|
1513
|
+
|
|
1514
|
+
/// \brief Set the format field of a union schema
|
|
1515
|
+
///
|
|
1516
|
+
/// Returns EINVAL for a type that is not NANOARROW_TYPE_DENSE_UNION
|
|
1517
|
+
/// or NANOARROW_TYPE_SPARSE_UNION. The specified number of children are
|
|
1518
|
+
/// allocated, and initialized.
|
|
1519
|
+
NANOARROW_DLL ArrowErrorCode ArrowSchemaSetTypeUnion(struct ArrowSchema* schema,
|
|
1520
|
+
enum ArrowType type,
|
|
1521
|
+
int64_t n_children);
|
|
1522
|
+
|
|
1523
|
+
/// \brief Make a (recursive) copy of a schema
|
|
1524
|
+
///
|
|
1525
|
+
/// Allocates and copies fields of schema into schema_out.
|
|
1526
|
+
NANOARROW_DLL ArrowErrorCode ArrowSchemaDeepCopy(const struct ArrowSchema* schema,
|
|
1527
|
+
struct ArrowSchema* schema_out);
|
|
1528
|
+
|
|
1529
|
+
/// \brief Copy format into schema->format
|
|
1530
|
+
///
|
|
1531
|
+
/// schema must have been allocated using ArrowSchemaInitFromType() or
|
|
1532
|
+
/// ArrowSchemaDeepCopy().
|
|
1533
|
+
NANOARROW_DLL ArrowErrorCode ArrowSchemaSetFormat(struct ArrowSchema* schema,
|
|
1534
|
+
const char* format);
|
|
1535
|
+
|
|
1536
|
+
/// \brief Copy name into schema->name
|
|
1537
|
+
///
|
|
1538
|
+
/// schema must have been allocated using ArrowSchemaInitFromType() or
|
|
1539
|
+
/// ArrowSchemaDeepCopy().
|
|
1540
|
+
NANOARROW_DLL ArrowErrorCode ArrowSchemaSetName(struct ArrowSchema* schema,
|
|
1541
|
+
const char* name);
|
|
1542
|
+
|
|
1543
|
+
/// \brief Copy metadata into schema->metadata
|
|
1544
|
+
///
|
|
1545
|
+
/// schema must have been allocated using ArrowSchemaInitFromType() or
|
|
1546
|
+
/// ArrowSchemaDeepCopy.
|
|
1547
|
+
NANOARROW_DLL ArrowErrorCode ArrowSchemaSetMetadata(struct ArrowSchema* schema,
|
|
1548
|
+
const char* metadata);
|
|
1549
|
+
|
|
1550
|
+
/// \brief Allocate the schema->children array
|
|
1551
|
+
///
|
|
1552
|
+
/// Includes the memory for each child struct ArrowSchema.
|
|
1553
|
+
/// schema must have been allocated using ArrowSchemaInitFromType() or
|
|
1554
|
+
/// ArrowSchemaDeepCopy().
|
|
1555
|
+
NANOARROW_DLL ArrowErrorCode ArrowSchemaAllocateChildren(struct ArrowSchema* schema,
|
|
1556
|
+
int64_t n_children);
|
|
1557
|
+
|
|
1558
|
+
/// \brief Allocate the schema->dictionary member
|
|
1559
|
+
///
|
|
1560
|
+
/// schema must have been allocated using ArrowSchemaInitFromType() or
|
|
1561
|
+
/// ArrowSchemaDeepCopy().
|
|
1562
|
+
NANOARROW_DLL ArrowErrorCode ArrowSchemaAllocateDictionary(struct ArrowSchema* schema);
|
|
1563
|
+
|
|
1564
|
+
/// @}
|
|
1565
|
+
|
|
1566
|
+
/// \defgroup nanoarrow-metadata Create, read, and modify schema metadata
|
|
1567
|
+
///
|
|
1568
|
+
/// @{
|
|
1569
|
+
|
|
1570
|
+
/// \brief Reader for key/value pairs in schema metadata
|
|
1571
|
+
///
|
|
1572
|
+
/// The ArrowMetadataReader does not own any data and is only valid
|
|
1573
|
+
/// for the lifetime of the underlying metadata pointer.
|
|
1574
|
+
struct ArrowMetadataReader {
|
|
1575
|
+
/// \brief A metadata string from a schema->metadata field.
|
|
1576
|
+
const char* metadata;
|
|
1577
|
+
|
|
1578
|
+
/// \brief The current offset into the metadata string
|
|
1579
|
+
int64_t offset;
|
|
1580
|
+
|
|
1581
|
+
/// \brief The number of remaining keys
|
|
1582
|
+
int32_t remaining_keys;
|
|
1583
|
+
};
|
|
1584
|
+
|
|
1585
|
+
/// \brief Initialize an ArrowMetadataReader
|
|
1586
|
+
NANOARROW_DLL ArrowErrorCode ArrowMetadataReaderInit(struct ArrowMetadataReader* reader,
|
|
1587
|
+
const char* metadata);
|
|
1588
|
+
|
|
1589
|
+
/// \brief Read the next key/value pair from an ArrowMetadataReader
|
|
1590
|
+
NANOARROW_DLL ArrowErrorCode ArrowMetadataReaderRead(struct ArrowMetadataReader* reader,
|
|
1591
|
+
struct ArrowStringView* key_out,
|
|
1592
|
+
struct ArrowStringView* value_out);
|
|
1593
|
+
|
|
1594
|
+
/// \brief The number of bytes in in a key/value metadata string
|
|
1595
|
+
NANOARROW_DLL int64_t ArrowMetadataSizeOf(const char* metadata);
|
|
1596
|
+
|
|
1597
|
+
/// \brief Check for a key in schema metadata
|
|
1598
|
+
NANOARROW_DLL char ArrowMetadataHasKey(const char* metadata, struct ArrowStringView key);
|
|
1599
|
+
|
|
1600
|
+
/// \brief Extract a value from schema metadata
|
|
1601
|
+
///
|
|
1602
|
+
/// If key does not exist in metadata, value_out is unmodified
|
|
1603
|
+
NANOARROW_DLL ArrowErrorCode ArrowMetadataGetValue(const char* metadata,
|
|
1604
|
+
struct ArrowStringView key,
|
|
1605
|
+
struct ArrowStringView* value_out);
|
|
1606
|
+
|
|
1607
|
+
/// \brief Initialize a builder for schema metadata from key/value pairs
|
|
1608
|
+
///
|
|
1609
|
+
/// metadata can be an existing metadata string or NULL to initialize
|
|
1610
|
+
/// an empty metadata string.
|
|
1611
|
+
NANOARROW_DLL ArrowErrorCode ArrowMetadataBuilderInit(struct ArrowBuffer* buffer,
|
|
1612
|
+
const char* metadata);
|
|
1613
|
+
|
|
1614
|
+
/// \brief Append a key/value pair to a buffer containing serialized metadata
|
|
1615
|
+
NANOARROW_DLL ArrowErrorCode ArrowMetadataBuilderAppend(struct ArrowBuffer* buffer,
|
|
1616
|
+
struct ArrowStringView key,
|
|
1617
|
+
struct ArrowStringView value);
|
|
1618
|
+
|
|
1619
|
+
/// \brief Set a key/value pair to a buffer containing serialized metadata
|
|
1620
|
+
///
|
|
1621
|
+
/// Ensures that the only entry for key in the metadata is set to value.
|
|
1622
|
+
/// This function maintains the existing position of (the first instance of)
|
|
1623
|
+
/// key if present in the data.
|
|
1624
|
+
NANOARROW_DLL ArrowErrorCode ArrowMetadataBuilderSet(struct ArrowBuffer* buffer,
|
|
1625
|
+
struct ArrowStringView key,
|
|
1626
|
+
struct ArrowStringView value);
|
|
1627
|
+
|
|
1628
|
+
/// \brief Remove a key from a buffer containing serialized metadata
|
|
1629
|
+
NANOARROW_DLL ArrowErrorCode ArrowMetadataBuilderRemove(struct ArrowBuffer* buffer,
|
|
1630
|
+
struct ArrowStringView key);
|
|
1631
|
+
|
|
1632
|
+
/// @}
|
|
1633
|
+
|
|
1634
|
+
/// \defgroup nanoarrow-schema-view Reading schemas
|
|
1635
|
+
///
|
|
1636
|
+
/// @{
|
|
1637
|
+
|
|
1638
|
+
/// \brief A non-owning view of a parsed ArrowSchema
|
|
1639
|
+
///
|
|
1640
|
+
/// Contains more readily extractable values than a raw ArrowSchema.
|
|
1641
|
+
/// Clients can stack or statically allocate this structure but are
|
|
1642
|
+
/// encouraged to use the provided getters to ensure forward
|
|
1643
|
+
/// compatibility.
|
|
1644
|
+
struct ArrowSchemaView {
|
|
1645
|
+
/// \brief A pointer to the schema represented by this view
|
|
1646
|
+
const struct ArrowSchema* schema;
|
|
1647
|
+
|
|
1648
|
+
/// \brief The data type represented by the schema
|
|
1649
|
+
///
|
|
1650
|
+
/// This value may be NANOARROW_TYPE_DICTIONARY if the schema has a
|
|
1651
|
+
/// non-null dictionary member; datetime types are valid values.
|
|
1652
|
+
/// This value will never be NANOARROW_TYPE_EXTENSION (see
|
|
1653
|
+
/// extension_name and/or extension_metadata to check for
|
|
1654
|
+
/// an extension type).
|
|
1655
|
+
enum ArrowType type;
|
|
1656
|
+
|
|
1657
|
+
/// \brief The storage data type represented by the schema
|
|
1658
|
+
///
|
|
1659
|
+
/// This value will never be NANOARROW_TYPE_DICTIONARY, NANOARROW_TYPE_EXTENSION
|
|
1660
|
+
/// or any datetime type. This value represents only the type required to
|
|
1661
|
+
/// interpret the buffers in the array.
|
|
1662
|
+
enum ArrowType storage_type;
|
|
1663
|
+
|
|
1664
|
+
/// \brief The storage layout represented by the schema
|
|
1665
|
+
struct ArrowLayout layout;
|
|
1666
|
+
|
|
1667
|
+
/// \brief The extension type name if it exists
|
|
1668
|
+
///
|
|
1669
|
+
/// If the ARROW:extension:name key is present in schema.metadata,
|
|
1670
|
+
/// extension_name.data will be non-NULL.
|
|
1671
|
+
struct ArrowStringView extension_name;
|
|
1672
|
+
|
|
1673
|
+
/// \brief The extension type metadata if it exists
|
|
1674
|
+
///
|
|
1675
|
+
/// If the ARROW:extension:metadata key is present in schema.metadata,
|
|
1676
|
+
/// extension_metadata.data will be non-NULL.
|
|
1677
|
+
struct ArrowStringView extension_metadata;
|
|
1678
|
+
|
|
1679
|
+
/// \brief Format fixed size parameter
|
|
1680
|
+
///
|
|
1681
|
+
/// This value is set when parsing a fixed-size binary or fixed-size
|
|
1682
|
+
/// list schema; this value is undefined for other types. For a
|
|
1683
|
+
/// fixed-size binary schema this value is in bytes; for a fixed-size
|
|
1684
|
+
/// list schema this value refers to the number of child elements for
|
|
1685
|
+
/// each element of the parent.
|
|
1686
|
+
int32_t fixed_size;
|
|
1687
|
+
|
|
1688
|
+
/// \brief Decimal bitwidth
|
|
1689
|
+
///
|
|
1690
|
+
/// This value is set when parsing a decimal type schema;
|
|
1691
|
+
/// this value is undefined for other types.
|
|
1692
|
+
int32_t decimal_bitwidth;
|
|
1693
|
+
|
|
1694
|
+
/// \brief Decimal precision
|
|
1695
|
+
///
|
|
1696
|
+
/// This value is set when parsing a decimal type schema;
|
|
1697
|
+
/// this value is undefined for other types.
|
|
1698
|
+
int32_t decimal_precision;
|
|
1699
|
+
|
|
1700
|
+
/// \brief Decimal scale
|
|
1701
|
+
///
|
|
1702
|
+
/// This value is set when parsing a decimal type schema;
|
|
1703
|
+
/// this value is undefined for other types.
|
|
1704
|
+
int32_t decimal_scale;
|
|
1705
|
+
|
|
1706
|
+
/// \brief Format time unit parameter
|
|
1707
|
+
///
|
|
1708
|
+
/// This value is set when parsing a date/time type. The value is
|
|
1709
|
+
/// undefined for other types.
|
|
1710
|
+
enum ArrowTimeUnit time_unit;
|
|
1711
|
+
|
|
1712
|
+
/// \brief Format timezone parameter
|
|
1713
|
+
///
|
|
1714
|
+
/// This value is set when parsing a timestamp type and represents
|
|
1715
|
+
/// the timezone format parameter. This value points to
|
|
1716
|
+
/// data within the schema and is undefined for other types.
|
|
1717
|
+
const char* timezone;
|
|
1718
|
+
|
|
1719
|
+
/// \brief Union type ids parameter
|
|
1720
|
+
///
|
|
1721
|
+
/// This value is set when parsing a union type and represents
|
|
1722
|
+
/// type ids parameter. This value points to
|
|
1723
|
+
/// data within the schema and is undefined for other types.
|
|
1724
|
+
const char* union_type_ids;
|
|
1725
|
+
};
|
|
1726
|
+
|
|
1727
|
+
/// \brief Initialize an ArrowSchemaView
|
|
1728
|
+
NANOARROW_DLL ArrowErrorCode ArrowSchemaViewInit(struct ArrowSchemaView* schema_view,
|
|
1729
|
+
const struct ArrowSchema* schema,
|
|
1730
|
+
struct ArrowError* error);
|
|
1731
|
+
|
|
1732
|
+
/// @}
|
|
1733
|
+
|
|
1734
|
+
/// \defgroup nanoarrow-buffer Owning, growable buffers
|
|
1735
|
+
///
|
|
1736
|
+
/// @{
|
|
1737
|
+
|
|
1738
|
+
/// \brief Initialize an ArrowBuffer
|
|
1739
|
+
///
|
|
1740
|
+
/// Initialize a buffer with a NULL, zero-size buffer using the default
|
|
1741
|
+
/// buffer allocator.
|
|
1742
|
+
static inline void ArrowBufferInit(struct ArrowBuffer* buffer);
|
|
1743
|
+
|
|
1744
|
+
/// \brief Set a newly-initialized buffer's allocator
|
|
1745
|
+
///
|
|
1746
|
+
/// Returns EINVAL if the buffer has already been allocated.
|
|
1747
|
+
static inline ArrowErrorCode ArrowBufferSetAllocator(
|
|
1748
|
+
struct ArrowBuffer* buffer, struct ArrowBufferAllocator allocator);
|
|
1749
|
+
|
|
1750
|
+
/// \brief Reset an ArrowBuffer
|
|
1751
|
+
///
|
|
1752
|
+
/// Releases the buffer using the allocator's free method if
|
|
1753
|
+
/// the buffer's data member is non-null, sets the data member
|
|
1754
|
+
/// to NULL, and sets the buffer's size and capacity to 0.
|
|
1755
|
+
static inline void ArrowBufferReset(struct ArrowBuffer* buffer);
|
|
1756
|
+
|
|
1757
|
+
/// \brief Move an ArrowBuffer
|
|
1758
|
+
///
|
|
1759
|
+
/// Transfers the buffer data and lifecycle management to another
|
|
1760
|
+
/// address and resets buffer.
|
|
1761
|
+
static inline void ArrowBufferMove(struct ArrowBuffer* src, struct ArrowBuffer* dst);
|
|
1762
|
+
|
|
1763
|
+
/// \brief Grow or shrink a buffer to a given size
|
|
1764
|
+
///
|
|
1765
|
+
/// When shrinking the size of the buffer, the buffer is only reallocated
|
|
1766
|
+
/// if shrink_to_fit is non-zero.
|
|
1767
|
+
static inline ArrowErrorCode ArrowBufferResize(struct ArrowBuffer* buffer,
|
|
1768
|
+
int64_t new_size_bytes,
|
|
1769
|
+
char shrink_to_fit);
|
|
1770
|
+
|
|
1771
|
+
/// \brief Ensure a buffer has at least a given additional capacity
|
|
1772
|
+
///
|
|
1773
|
+
/// Ensures that the buffer has space to append at least
|
|
1774
|
+
/// additional_size_bytes, overallocating when required.
|
|
1775
|
+
static inline ArrowErrorCode ArrowBufferReserve(struct ArrowBuffer* buffer,
|
|
1776
|
+
int64_t additional_size_bytes);
|
|
1777
|
+
|
|
1778
|
+
/// \brief Write data to buffer and increment the buffer size
|
|
1779
|
+
///
|
|
1780
|
+
/// This function does not check that buffer has the required capacity
|
|
1781
|
+
static inline void ArrowBufferAppendUnsafe(struct ArrowBuffer* buffer, const void* data,
|
|
1782
|
+
int64_t size_bytes);
|
|
1783
|
+
|
|
1784
|
+
/// \brief Write data to buffer and increment the buffer size
|
|
1785
|
+
///
|
|
1786
|
+
/// This function writes and ensures that the buffer has the required capacity,
|
|
1787
|
+
/// possibly by reallocating the buffer. Like ArrowBufferReserve, this will
|
|
1788
|
+
/// overallocate when reallocation is required.
|
|
1789
|
+
static inline ArrowErrorCode ArrowBufferAppend(struct ArrowBuffer* buffer,
|
|
1790
|
+
const void* data, int64_t size_bytes);
|
|
1791
|
+
|
|
1792
|
+
/// \brief Write fill to buffer and increment the buffer size
|
|
1793
|
+
///
|
|
1794
|
+
/// This function writes the specified number of fill bytes and
|
|
1795
|
+
/// ensures that the buffer has the required capacity,
|
|
1796
|
+
static inline ArrowErrorCode ArrowBufferAppendFill(struct ArrowBuffer* buffer,
|
|
1797
|
+
uint8_t value, int64_t size_bytes);
|
|
1798
|
+
|
|
1799
|
+
/// \brief Write an 8-bit integer to a buffer
|
|
1800
|
+
static inline ArrowErrorCode ArrowBufferAppendInt8(struct ArrowBuffer* buffer,
|
|
1801
|
+
int8_t value);
|
|
1802
|
+
|
|
1803
|
+
/// \brief Write an unsigned 8-bit integer to a buffer
|
|
1804
|
+
static inline ArrowErrorCode ArrowBufferAppendUInt8(struct ArrowBuffer* buffer,
|
|
1805
|
+
uint8_t value);
|
|
1806
|
+
|
|
1807
|
+
/// \brief Write a 16-bit integer to a buffer
|
|
1808
|
+
static inline ArrowErrorCode ArrowBufferAppendInt16(struct ArrowBuffer* buffer,
|
|
1809
|
+
int16_t value);
|
|
1810
|
+
|
|
1811
|
+
/// \brief Write an unsigned 16-bit integer to a buffer
|
|
1812
|
+
static inline ArrowErrorCode ArrowBufferAppendUInt16(struct ArrowBuffer* buffer,
|
|
1813
|
+
uint16_t value);
|
|
1814
|
+
|
|
1815
|
+
/// \brief Write a 32-bit integer to a buffer
|
|
1816
|
+
static inline ArrowErrorCode ArrowBufferAppendInt32(struct ArrowBuffer* buffer,
|
|
1817
|
+
int32_t value);
|
|
1818
|
+
|
|
1819
|
+
/// \brief Write an unsigned 32-bit integer to a buffer
|
|
1820
|
+
static inline ArrowErrorCode ArrowBufferAppendUInt32(struct ArrowBuffer* buffer,
|
|
1821
|
+
uint32_t value);
|
|
1822
|
+
|
|
1823
|
+
/// \brief Write a 64-bit integer to a buffer
|
|
1824
|
+
static inline ArrowErrorCode ArrowBufferAppendInt64(struct ArrowBuffer* buffer,
|
|
1825
|
+
int64_t value);
|
|
1826
|
+
|
|
1827
|
+
/// \brief Write an unsigned 64-bit integer to a buffer
|
|
1828
|
+
static inline ArrowErrorCode ArrowBufferAppendUInt64(struct ArrowBuffer* buffer,
|
|
1829
|
+
uint64_t value);
|
|
1830
|
+
|
|
1831
|
+
/// \brief Write a double to a buffer
|
|
1832
|
+
static inline ArrowErrorCode ArrowBufferAppendDouble(struct ArrowBuffer* buffer,
|
|
1833
|
+
double value);
|
|
1834
|
+
|
|
1835
|
+
/// \brief Write a float to a buffer
|
|
1836
|
+
static inline ArrowErrorCode ArrowBufferAppendFloat(struct ArrowBuffer* buffer,
|
|
1837
|
+
float value);
|
|
1838
|
+
|
|
1839
|
+
/// \brief Write an ArrowStringView to a buffer
|
|
1840
|
+
static inline ArrowErrorCode ArrowBufferAppendStringView(struct ArrowBuffer* buffer,
|
|
1841
|
+
struct ArrowStringView value);
|
|
1842
|
+
|
|
1843
|
+
/// \brief Write an ArrowBufferView to a buffer
|
|
1844
|
+
static inline ArrowErrorCode ArrowBufferAppendBufferView(struct ArrowBuffer* buffer,
|
|
1845
|
+
struct ArrowBufferView value);
|
|
1846
|
+
|
|
1847
|
+
/// @}
|
|
1848
|
+
|
|
1849
|
+
/// \defgroup nanoarrow-bitmap Bitmap utilities
|
|
1850
|
+
///
|
|
1851
|
+
/// @{
|
|
1852
|
+
|
|
1853
|
+
/// \brief Extract a boolean value from a bitmap
|
|
1854
|
+
static inline int8_t ArrowBitGet(const uint8_t* bits, int64_t i);
|
|
1855
|
+
|
|
1856
|
+
/// \brief Set a boolean value to a bitmap to true
|
|
1857
|
+
static inline void ArrowBitSet(uint8_t* bits, int64_t i);
|
|
1858
|
+
|
|
1859
|
+
/// \brief Set a boolean value to a bitmap to false
|
|
1860
|
+
static inline void ArrowBitClear(uint8_t* bits, int64_t i);
|
|
1861
|
+
|
|
1862
|
+
/// \brief Set a boolean value to a bitmap
|
|
1863
|
+
static inline void ArrowBitSetTo(uint8_t* bits, int64_t i, uint8_t value);
|
|
1864
|
+
|
|
1865
|
+
/// \brief Set a boolean value to a range in a bitmap
|
|
1866
|
+
static inline void ArrowBitsSetTo(uint8_t* bits, int64_t start_offset, int64_t length,
|
|
1867
|
+
uint8_t bits_are_set);
|
|
1868
|
+
|
|
1869
|
+
/// \brief Count true values in a bitmap
|
|
1870
|
+
static inline int64_t ArrowBitCountSet(const uint8_t* bits, int64_t i_from, int64_t i_to);
|
|
1871
|
+
|
|
1872
|
+
/// \brief Extract int8 boolean values from a range in a bitmap
|
|
1873
|
+
static inline void ArrowBitsUnpackInt8(const uint8_t* bits, int64_t start_offset,
|
|
1874
|
+
int64_t length, int8_t* out);
|
|
1875
|
+
|
|
1876
|
+
/// \brief Extract int32 boolean values from a range in a bitmap
|
|
1877
|
+
static inline void ArrowBitsUnpackInt32(const uint8_t* bits, int64_t start_offset,
|
|
1878
|
+
int64_t length, int32_t* out);
|
|
1879
|
+
|
|
1880
|
+
/// \brief Initialize an ArrowBitmap
|
|
1881
|
+
///
|
|
1882
|
+
/// Initialize the builder's buffer, empty its cache, and reset the size to zero
|
|
1883
|
+
static inline void ArrowBitmapInit(struct ArrowBitmap* bitmap);
|
|
1884
|
+
|
|
1885
|
+
/// \brief Move an ArrowBitmap
|
|
1886
|
+
///
|
|
1887
|
+
/// Transfers the underlying buffer data and lifecycle management to another
|
|
1888
|
+
/// address and resets the bitmap.
|
|
1889
|
+
static inline void ArrowBitmapMove(struct ArrowBitmap* src, struct ArrowBitmap* dst);
|
|
1890
|
+
|
|
1891
|
+
/// \brief Ensure a bitmap builder has at least a given additional capacity
|
|
1892
|
+
///
|
|
1893
|
+
/// Ensures that the buffer has space to append at least
|
|
1894
|
+
/// additional_size_bits, overallocating when required.
|
|
1895
|
+
static inline ArrowErrorCode ArrowBitmapReserve(struct ArrowBitmap* bitmap,
|
|
1896
|
+
int64_t additional_size_bits);
|
|
1897
|
+
|
|
1898
|
+
/// \brief Grow or shrink a bitmap to a given size
|
|
1899
|
+
///
|
|
1900
|
+
/// When shrinking the size of the bitmap, the bitmap is only reallocated
|
|
1901
|
+
/// if shrink_to_fit is non-zero.
|
|
1902
|
+
static inline ArrowErrorCode ArrowBitmapResize(struct ArrowBitmap* bitmap,
|
|
1903
|
+
int64_t new_size_bits, char shrink_to_fit);
|
|
1904
|
+
|
|
1905
|
+
/// \brief Reserve space for and append zero or more of the same boolean value to a bitmap
|
|
1906
|
+
static inline ArrowErrorCode ArrowBitmapAppend(struct ArrowBitmap* bitmap,
|
|
1907
|
+
uint8_t bits_are_set, int64_t length);
|
|
1908
|
+
|
|
1909
|
+
/// \brief Append zero or more of the same boolean value to a bitmap
|
|
1910
|
+
static inline void ArrowBitmapAppendUnsafe(struct ArrowBitmap* bitmap,
|
|
1911
|
+
uint8_t bits_are_set, int64_t length);
|
|
1912
|
+
|
|
1913
|
+
/// \brief Append boolean values encoded as int8_t to a bitmap
|
|
1914
|
+
///
|
|
1915
|
+
/// The values must all be 0 or 1.
|
|
1916
|
+
static inline void ArrowBitmapAppendInt8Unsafe(struct ArrowBitmap* bitmap,
|
|
1917
|
+
const int8_t* values, int64_t n_values);
|
|
1918
|
+
|
|
1919
|
+
/// \brief Append boolean values encoded as int32_t to a bitmap
|
|
1920
|
+
///
|
|
1921
|
+
/// The values must all be 0 or 1.
|
|
1922
|
+
static inline void ArrowBitmapAppendInt32Unsafe(struct ArrowBitmap* bitmap,
|
|
1923
|
+
const int32_t* values, int64_t n_values);
|
|
1924
|
+
|
|
1925
|
+
/// \brief Reset a bitmap builder
|
|
1926
|
+
///
|
|
1927
|
+
/// Releases any memory held by buffer, empties the cache, and resets the size to zero
|
|
1928
|
+
static inline void ArrowBitmapReset(struct ArrowBitmap* bitmap);
|
|
1929
|
+
|
|
1930
|
+
/// @}
|
|
1931
|
+
|
|
1932
|
+
/// \defgroup nanoarrow-array Creating arrays
|
|
1933
|
+
///
|
|
1934
|
+
/// These functions allocate, copy, and destroy ArrowArray structures.
|
|
1935
|
+
/// Once an ArrowArray has been initialized via ArrowArrayInitFromType()
|
|
1936
|
+
/// or ArrowArrayInitFromSchema(), the caller is responsible for releasing
|
|
1937
|
+
/// it using the embedded release callback.
|
|
1938
|
+
///
|
|
1939
|
+
/// @{
|
|
1940
|
+
|
|
1941
|
+
/// \brief Initialize the fields of an array
|
|
1942
|
+
///
|
|
1943
|
+
/// Initializes the fields and release callback of array. Caller
|
|
1944
|
+
/// is responsible for calling the array->release callback if
|
|
1945
|
+
/// NANOARROW_OK is returned.
|
|
1946
|
+
NANOARROW_DLL ArrowErrorCode ArrowArrayInitFromType(struct ArrowArray* array,
|
|
1947
|
+
enum ArrowType storage_type);
|
|
1948
|
+
|
|
1949
|
+
/// \brief Initialize the contents of an ArrowArray from an ArrowSchema
|
|
1950
|
+
///
|
|
1951
|
+
/// Caller is responsible for calling the array->release callback if
|
|
1952
|
+
/// NANOARROW_OK is returned.
|
|
1953
|
+
NANOARROW_DLL ArrowErrorCode ArrowArrayInitFromSchema(struct ArrowArray* array,
|
|
1954
|
+
const struct ArrowSchema* schema,
|
|
1955
|
+
struct ArrowError* error);
|
|
1956
|
+
|
|
1957
|
+
/// \brief Initialize the contents of an ArrowArray from an ArrowArrayView
|
|
1958
|
+
///
|
|
1959
|
+
/// Caller is responsible for calling the array->release callback if
|
|
1960
|
+
/// NANOARROW_OK is returned.
|
|
1961
|
+
NANOARROW_DLL ArrowErrorCode ArrowArrayInitFromArrayView(
|
|
1962
|
+
struct ArrowArray* array, const struct ArrowArrayView* array_view,
|
|
1963
|
+
struct ArrowError* error);
|
|
1964
|
+
|
|
1965
|
+
/// \brief Allocate the array->children array
|
|
1966
|
+
///
|
|
1967
|
+
/// Includes the memory for each child struct ArrowArray,
|
|
1968
|
+
/// whose members are marked as released and may be subsequently initialized
|
|
1969
|
+
/// with ArrowArrayInitFromType() or moved from an existing ArrowArray.
|
|
1970
|
+
/// schema must have been allocated using ArrowArrayInitFromType().
|
|
1971
|
+
NANOARROW_DLL ArrowErrorCode ArrowArrayAllocateChildren(struct ArrowArray* array,
|
|
1972
|
+
int64_t n_children);
|
|
1973
|
+
|
|
1974
|
+
/// \brief Allocate the array->dictionary member
|
|
1975
|
+
///
|
|
1976
|
+
/// Includes the memory for the struct ArrowArray, whose contents
|
|
1977
|
+
/// is marked as released and may be subsequently initialized
|
|
1978
|
+
/// with ArrowArrayInitFromType() or moved from an existing ArrowArray.
|
|
1979
|
+
/// array must have been allocated using ArrowArrayInitFromType()
|
|
1980
|
+
NANOARROW_DLL ArrowErrorCode ArrowArrayAllocateDictionary(struct ArrowArray* array);
|
|
1981
|
+
|
|
1982
|
+
/// \brief Set the validity bitmap of an ArrowArray
|
|
1983
|
+
///
|
|
1984
|
+
/// array must have been allocated using ArrowArrayInitFromType()
|
|
1985
|
+
NANOARROW_DLL void ArrowArraySetValidityBitmap(struct ArrowArray* array,
|
|
1986
|
+
struct ArrowBitmap* bitmap);
|
|
1987
|
+
|
|
1988
|
+
/// \brief Set a buffer of an ArrowArray
|
|
1989
|
+
///
|
|
1990
|
+
/// array must have been allocated using ArrowArrayInitFromType()
|
|
1991
|
+
NANOARROW_DLL ArrowErrorCode ArrowArraySetBuffer(struct ArrowArray* array, int64_t i,
|
|
1992
|
+
struct ArrowBuffer* buffer);
|
|
1993
|
+
|
|
1994
|
+
/// \brief Add variadic buffers to a string or binary view array
|
|
1995
|
+
///
|
|
1996
|
+
/// array must have been allocated using ArrowArrayInitFromType()
|
|
1997
|
+
static inline ArrowErrorCode ArrowArrayAddVariadicBuffers(struct ArrowArray* array,
|
|
1998
|
+
int32_t n_buffers);
|
|
1999
|
+
|
|
2000
|
+
/// \brief Get the validity bitmap of an ArrowArray
|
|
2001
|
+
///
|
|
2002
|
+
/// array must have been allocated using ArrowArrayInitFromType()
|
|
2003
|
+
static inline struct ArrowBitmap* ArrowArrayValidityBitmap(struct ArrowArray* array);
|
|
2004
|
+
|
|
2005
|
+
/// \brief Get a buffer of an ArrowArray
|
|
2006
|
+
///
|
|
2007
|
+
/// array must have been allocated using ArrowArrayInitFromType()
|
|
2008
|
+
static inline struct ArrowBuffer* ArrowArrayBuffer(struct ArrowArray* array, int64_t i);
|
|
2009
|
+
|
|
2010
|
+
/// \brief Start element-wise appending to an ArrowArray
|
|
2011
|
+
///
|
|
2012
|
+
/// Initializes any values needed to use ArrowArrayAppend*() functions.
|
|
2013
|
+
/// All element-wise appenders append by value and return EINVAL if the exact value
|
|
2014
|
+
/// cannot be represented by the underlying storage type.
|
|
2015
|
+
/// array must have been allocated using ArrowArrayInitFromType()
|
|
2016
|
+
static inline ArrowErrorCode ArrowArrayStartAppending(struct ArrowArray* array);
|
|
2017
|
+
|
|
2018
|
+
/// \brief Reserve space for future appends
|
|
2019
|
+
///
|
|
2020
|
+
/// For buffer sizes that can be calculated (i.e., not string data buffers or
|
|
2021
|
+
/// child array sizes for non-fixed-size arrays), recursively reserve space for
|
|
2022
|
+
/// additional elements. This is useful for reducing the number of reallocations
|
|
2023
|
+
/// that occur using the item-wise appenders.
|
|
2024
|
+
NANOARROW_DLL ArrowErrorCode ArrowArrayReserve(struct ArrowArray* array,
|
|
2025
|
+
int64_t additional_size_elements);
|
|
2026
|
+
|
|
2027
|
+
/// \brief Append a null value to an array
|
|
2028
|
+
static inline ArrowErrorCode ArrowArrayAppendNull(struct ArrowArray* array, int64_t n);
|
|
2029
|
+
|
|
2030
|
+
/// \brief Append an empty, non-null value to an array
|
|
2031
|
+
static inline ArrowErrorCode ArrowArrayAppendEmpty(struct ArrowArray* array, int64_t n);
|
|
2032
|
+
|
|
2033
|
+
/// \brief Append a signed integer value to an array
|
|
2034
|
+
///
|
|
2035
|
+
/// Returns NANOARROW_OK if value can be exactly represented by
|
|
2036
|
+
/// the underlying storage type or EINVAL otherwise (e.g., value
|
|
2037
|
+
/// is outside the valid array range).
|
|
2038
|
+
static inline ArrowErrorCode ArrowArrayAppendInt(struct ArrowArray* array, int64_t value);
|
|
2039
|
+
|
|
2040
|
+
/// \brief Append an unsigned integer value to an array
|
|
2041
|
+
///
|
|
2042
|
+
/// Returns NANOARROW_OK if value can be exactly represented by
|
|
2043
|
+
/// the underlying storage type or EINVAL otherwise (e.g., value
|
|
2044
|
+
/// is outside the valid array range).
|
|
2045
|
+
static inline ArrowErrorCode ArrowArrayAppendUInt(struct ArrowArray* array,
|
|
2046
|
+
uint64_t value);
|
|
2047
|
+
|
|
2048
|
+
/// \brief Append a double value to an array
|
|
2049
|
+
///
|
|
2050
|
+
/// Returns NANOARROW_OK if value can be exactly represented by
|
|
2051
|
+
/// the underlying storage type or EINVAL otherwise (e.g., value
|
|
2052
|
+
/// is outside the valid array range or there is an attempt to append
|
|
2053
|
+
/// a non-integer to an array with an integer storage type).
|
|
2054
|
+
static inline ArrowErrorCode ArrowArrayAppendDouble(struct ArrowArray* array,
|
|
2055
|
+
double value);
|
|
2056
|
+
|
|
2057
|
+
/// \brief Append a string of bytes to an array
|
|
2058
|
+
///
|
|
2059
|
+
/// Returns NANOARROW_OK if value can be exactly represented by
|
|
2060
|
+
/// the underlying storage type, EOVERFLOW if appending value would overflow
|
|
2061
|
+
/// the offset type (e.g., if the data buffer would be larger than 2 GB for a
|
|
2062
|
+
/// non-large string type), or EINVAL otherwise (e.g., the underlying array is not a
|
|
2063
|
+
/// binary, string, large binary, large string, or fixed-size binary array, or value is
|
|
2064
|
+
/// the wrong size for a fixed-size binary array).
|
|
2065
|
+
static inline ArrowErrorCode ArrowArrayAppendBytes(struct ArrowArray* array,
|
|
2066
|
+
struct ArrowBufferView value);
|
|
2067
|
+
|
|
2068
|
+
/// \brief Append a string value to an array
|
|
2069
|
+
///
|
|
2070
|
+
/// Returns NANOARROW_OK if value can be exactly represented by
|
|
2071
|
+
/// the underlying storage type, EOVERFLOW if appending value would overflow
|
|
2072
|
+
/// the offset type (e.g., if the data buffer would be larger than 2 GB for a
|
|
2073
|
+
/// non-large string type), or EINVAL otherwise (e.g., the underlying array is not a
|
|
2074
|
+
/// string or large string array).
|
|
2075
|
+
static inline ArrowErrorCode ArrowArrayAppendString(struct ArrowArray* array,
|
|
2076
|
+
struct ArrowStringView value);
|
|
2077
|
+
|
|
2078
|
+
/// \brief Append a Interval to an array
|
|
2079
|
+
///
|
|
2080
|
+
/// Returns NANOARROW_OK if value can be exactly represented by
|
|
2081
|
+
/// the underlying storage type or EINVAL otherwise.
|
|
2082
|
+
static inline ArrowErrorCode ArrowArrayAppendInterval(struct ArrowArray* array,
|
|
2083
|
+
const struct ArrowInterval* value);
|
|
2084
|
+
|
|
2085
|
+
/// \brief Append a decimal value to an array
|
|
2086
|
+
///
|
|
2087
|
+
/// Returns NANOARROW_OK if array is a decimal array with the appropriate
|
|
2088
|
+
/// bitwidth or EINVAL otherwise.
|
|
2089
|
+
static inline ArrowErrorCode ArrowArrayAppendDecimal(struct ArrowArray* array,
|
|
2090
|
+
const struct ArrowDecimal* value);
|
|
2091
|
+
|
|
2092
|
+
/// \brief Finish a nested array element
|
|
2093
|
+
///
|
|
2094
|
+
/// Appends a non-null element to the array based on the first child's current
|
|
2095
|
+
/// length. Returns NANOARROW_OK if the item was successfully added, EOVERFLOW
|
|
2096
|
+
/// if the child of a list or map array would exceed INT_MAX elements, or EINVAL
|
|
2097
|
+
/// if the underlying storage type is not a struct, list, large list, or fixed-size
|
|
2098
|
+
/// list, or if there was an attempt to add a struct or fixed-size list element where the
|
|
2099
|
+
/// length of the child array(s) did not match the expected length.
|
|
2100
|
+
static inline ArrowErrorCode ArrowArrayFinishElement(struct ArrowArray* array);
|
|
2101
|
+
|
|
2102
|
+
/// \brief Finish a union array element
|
|
2103
|
+
///
|
|
2104
|
+
/// Appends an element to the union type ids buffer and increments array->length.
|
|
2105
|
+
/// For sparse unions, up to one element is added to non type-id children. Returns
|
|
2106
|
+
/// EINVAL if the underlying storage type is not a union, if type_id is not valid,
|
|
2107
|
+
/// or if child sizes after appending are inconsistent.
|
|
2108
|
+
static inline ArrowErrorCode ArrowArrayFinishUnionElement(struct ArrowArray* array,
|
|
2109
|
+
int8_t type_id);
|
|
2110
|
+
|
|
2111
|
+
/// \brief Shrink buffer capacity to the size required
|
|
2112
|
+
///
|
|
2113
|
+
/// Also applies shrinking to any child arrays. array must have been allocated using
|
|
2114
|
+
/// ArrowArrayInitFromType
|
|
2115
|
+
static inline ArrowErrorCode ArrowArrayShrinkToFit(struct ArrowArray* array);
|
|
2116
|
+
|
|
2117
|
+
/// \brief Finish building an ArrowArray
|
|
2118
|
+
///
|
|
2119
|
+
/// Flushes any pointers from internal buffers that may have been reallocated
|
|
2120
|
+
/// into array->buffers and checks the actual size of the buffers
|
|
2121
|
+
/// against the expected size based on the final length.
|
|
2122
|
+
/// array must have been allocated using ArrowArrayInitFromType()
|
|
2123
|
+
NANOARROW_DLL ArrowErrorCode ArrowArrayFinishBuildingDefault(struct ArrowArray* array,
|
|
2124
|
+
struct ArrowError* error);
|
|
2125
|
+
|
|
2126
|
+
/// \brief Finish building an ArrowArray with explicit validation
|
|
2127
|
+
///
|
|
2128
|
+
/// Finish building with an explicit validation level. This could perform less validation
|
|
2129
|
+
/// (i.e. NANOARROW_VALIDATION_LEVEL_NONE or NANOARROW_VALIDATION_LEVEL_MINIMAL) if CPU
|
|
2130
|
+
/// buffer data access is not possible or more validation (i.e.,
|
|
2131
|
+
/// NANOARROW_VALIDATION_LEVEL_FULL) if buffer content was obtained from an untrusted or
|
|
2132
|
+
/// corruptible source.
|
|
2133
|
+
NANOARROW_DLL ArrowErrorCode ArrowArrayFinishBuilding(
|
|
2134
|
+
struct ArrowArray* array, enum ArrowValidationLevel validation_level,
|
|
2135
|
+
struct ArrowError* error);
|
|
2136
|
+
|
|
2137
|
+
/// @}
|
|
2138
|
+
|
|
2139
|
+
/// \defgroup nanoarrow-array-view Reading arrays
|
|
2140
|
+
///
|
|
2141
|
+
/// These functions read and validate the contents ArrowArray structures.
|
|
2142
|
+
///
|
|
2143
|
+
/// @{
|
|
2144
|
+
|
|
2145
|
+
/// \brief Initialize the contents of an ArrowArrayView
|
|
2146
|
+
NANOARROW_DLL void ArrowArrayViewInitFromType(struct ArrowArrayView* array_view,
|
|
2147
|
+
enum ArrowType storage_type);
|
|
2148
|
+
|
|
2149
|
+
/// \brief Move an ArrowArrayView
|
|
2150
|
+
///
|
|
2151
|
+
/// Transfers the ArrowArrayView data and lifecycle management to another
|
|
2152
|
+
/// address and resets the contents of src.
|
|
2153
|
+
static inline void ArrowArrayViewMove(struct ArrowArrayView* src,
|
|
2154
|
+
struct ArrowArrayView* dst);
|
|
2155
|
+
|
|
2156
|
+
/// \brief Initialize the contents of an ArrowArrayView from an ArrowSchema
|
|
2157
|
+
NANOARROW_DLL ArrowErrorCode
|
|
2158
|
+
ArrowArrayViewInitFromSchema(struct ArrowArrayView* array_view,
|
|
2159
|
+
const struct ArrowSchema* schema, struct ArrowError* error);
|
|
2160
|
+
|
|
2161
|
+
/// \brief Allocate the array_view->children array
|
|
2162
|
+
///
|
|
2163
|
+
/// Includes the memory for each child struct ArrowArrayView
|
|
2164
|
+
NANOARROW_DLL ArrowErrorCode
|
|
2165
|
+
ArrowArrayViewAllocateChildren(struct ArrowArrayView* array_view, int64_t n_children);
|
|
2166
|
+
|
|
2167
|
+
/// \brief Allocate array_view->dictionary
|
|
2168
|
+
NANOARROW_DLL ArrowErrorCode
|
|
2169
|
+
ArrowArrayViewAllocateDictionary(struct ArrowArrayView* array_view);
|
|
2170
|
+
|
|
2171
|
+
/// \brief Set data-independent buffer sizes from length
|
|
2172
|
+
NANOARROW_DLL void ArrowArrayViewSetLength(struct ArrowArrayView* array_view,
|
|
2173
|
+
int64_t length);
|
|
2174
|
+
|
|
2175
|
+
/// \brief Set buffer sizes and data pointers from an ArrowArray
|
|
2176
|
+
NANOARROW_DLL ArrowErrorCode ArrowArrayViewSetArray(struct ArrowArrayView* array_view,
|
|
2177
|
+
const struct ArrowArray* array,
|
|
2178
|
+
struct ArrowError* error);
|
|
2179
|
+
|
|
2180
|
+
/// \brief Set buffer sizes and data pointers from an ArrowArray except for those
|
|
2181
|
+
/// that require dereferencing buffer content.
|
|
2182
|
+
NANOARROW_DLL ArrowErrorCode
|
|
2183
|
+
ArrowArrayViewSetArrayMinimal(struct ArrowArrayView* array_view,
|
|
2184
|
+
const struct ArrowArray* array, struct ArrowError* error);
|
|
2185
|
+
|
|
2186
|
+
/// \brief Get the number of buffers
|
|
2187
|
+
///
|
|
2188
|
+
/// The number of buffers referred to by this ArrowArrayView. In may cases this can also
|
|
2189
|
+
/// be calculated from the ArrowLayout member of the ArrowArrayView or ArrowSchemaView;
|
|
2190
|
+
/// however, for binary view and string view types, the number of total buffers depends on
|
|
2191
|
+
/// the number of variadic buffers.
|
|
2192
|
+
static inline int64_t ArrowArrayViewGetNumBuffers(struct ArrowArrayView* array_view);
|
|
2193
|
+
|
|
2194
|
+
/// \brief Get a view of a specific buffer from an ArrowArrayView
|
|
2195
|
+
///
|
|
2196
|
+
/// This is the ArrowArrayView equivalent of ArrowArray::buffers[i] that includes
|
|
2197
|
+
/// size information (if known).
|
|
2198
|
+
static inline struct ArrowBufferView ArrowArrayViewGetBufferView(
|
|
2199
|
+
struct ArrowArrayView* array_view, int64_t i);
|
|
2200
|
+
|
|
2201
|
+
/// \brief Get the function of a specific buffer in an ArrowArrayView
|
|
2202
|
+
///
|
|
2203
|
+
/// In may cases this can also be obtained from the ArrowLayout member of the
|
|
2204
|
+
/// ArrowArrayView or ArrowSchemaView; however, for binary view and string view types,
|
|
2205
|
+
/// the function of each buffer may be different between two arrays of the same type
|
|
2206
|
+
/// depending on the number of variadic buffers.
|
|
2207
|
+
static inline enum ArrowBufferType ArrowArrayViewGetBufferType(
|
|
2208
|
+
struct ArrowArrayView* array_view, int64_t i);
|
|
2209
|
+
|
|
2210
|
+
/// \brief Get the data type of a specific buffer in an ArrowArrayView
|
|
2211
|
+
///
|
|
2212
|
+
/// In may cases this can also be obtained from the ArrowLayout member of the
|
|
2213
|
+
/// ArrowArrayView or ArrowSchemaView; however, for binary view and string view types,
|
|
2214
|
+
/// the data type of each buffer may be different between two arrays of the same type
|
|
2215
|
+
/// depending on the number of variadic buffers.
|
|
2216
|
+
static inline enum ArrowType ArrowArrayViewGetBufferDataType(
|
|
2217
|
+
struct ArrowArrayView* array_view, int64_t i);
|
|
2218
|
+
|
|
2219
|
+
/// \brief Get the element size (in bits) of a specific buffer in an ArrowArrayView
|
|
2220
|
+
///
|
|
2221
|
+
/// In may cases this can also be obtained from the ArrowLayout member of the
|
|
2222
|
+
/// ArrowArrayView or ArrowSchemaView; however, for binary view and string view types,
|
|
2223
|
+
/// the element width of each buffer may be different between two arrays of the same type
|
|
2224
|
+
/// depending on the number of variadic buffers.
|
|
2225
|
+
static inline int64_t ArrowArrayViewGetBufferElementSizeBits(
|
|
2226
|
+
struct ArrowArrayView* array_view, int64_t i);
|
|
2227
|
+
|
|
2228
|
+
/// \brief Performs checks on the content of an ArrowArrayView
|
|
2229
|
+
///
|
|
2230
|
+
/// If using ArrowArrayViewSetArray() to back array_view with an ArrowArray,
|
|
2231
|
+
/// the buffer sizes and some content (fist and last offset) have already
|
|
2232
|
+
/// been validated at the "default" level. If setting the buffer pointers
|
|
2233
|
+
/// and sizes otherwise, you may wish to perform checks at a different level. See
|
|
2234
|
+
/// documentation for ArrowValidationLevel for the details of checks performed
|
|
2235
|
+
/// at each level.
|
|
2236
|
+
NANOARROW_DLL ArrowErrorCode ArrowArrayViewValidate(
|
|
2237
|
+
struct ArrowArrayView* array_view, enum ArrowValidationLevel validation_level,
|
|
2238
|
+
struct ArrowError* error);
|
|
2239
|
+
|
|
2240
|
+
/// \brief Compare two ArrowArrayView objects for equality
|
|
2241
|
+
///
|
|
2242
|
+
/// Given two ArrowArrayView instances, place either 0 (not equal) and
|
|
2243
|
+
/// 1 (equal) at the address pointed to by out. If the comparison determines
|
|
2244
|
+
/// that actual and expected are not equal, a reason will be communicated via
|
|
2245
|
+
/// error if error is non-NULL.
|
|
2246
|
+
///
|
|
2247
|
+
/// Returns NANOARROW_OK if the comparison completed successfully.
|
|
2248
|
+
NANOARROW_DLL ArrowErrorCode ArrowArrayViewCompare(const struct ArrowArrayView* actual,
|
|
2249
|
+
const struct ArrowArrayView* expected,
|
|
2250
|
+
enum ArrowCompareLevel level, int* out,
|
|
2251
|
+
struct ArrowError* reason);
|
|
2252
|
+
|
|
2253
|
+
/// \brief Reset the contents of an ArrowArrayView and frees resources
|
|
2254
|
+
NANOARROW_DLL void ArrowArrayViewReset(struct ArrowArrayView* array_view);
|
|
2255
|
+
|
|
2256
|
+
/// \brief Check for a null element in an ArrowArrayView
|
|
2257
|
+
static inline int8_t ArrowArrayViewIsNull(const struct ArrowArrayView* array_view,
|
|
2258
|
+
int64_t i);
|
|
2259
|
+
|
|
2260
|
+
/// \brief Compute null count for an ArrowArrayView
|
|
2261
|
+
static inline int64_t ArrowArrayViewComputeNullCount(
|
|
2262
|
+
const struct ArrowArrayView* array_view);
|
|
2263
|
+
|
|
2264
|
+
/// \brief Get the type id of a union array element
|
|
2265
|
+
static inline int8_t ArrowArrayViewUnionTypeId(const struct ArrowArrayView* array_view,
|
|
2266
|
+
int64_t i);
|
|
2267
|
+
|
|
2268
|
+
/// \brief Get the child index of a union array element
|
|
2269
|
+
static inline int8_t ArrowArrayViewUnionChildIndex(
|
|
2270
|
+
const struct ArrowArrayView* array_view, int64_t i);
|
|
2271
|
+
|
|
2272
|
+
/// \brief Get the index to use into the relevant union child array
|
|
2273
|
+
static inline int64_t ArrowArrayViewUnionChildOffset(
|
|
2274
|
+
const struct ArrowArrayView* array_view, int64_t i);
|
|
2275
|
+
|
|
2276
|
+
/// \brief Get an element in an ArrowArrayView as an integer
|
|
2277
|
+
///
|
|
2278
|
+
/// This function does not check for null values, that values are actually integers, or
|
|
2279
|
+
/// that values are within a valid range for an int64.
|
|
2280
|
+
static inline int64_t ArrowArrayViewGetIntUnsafe(const struct ArrowArrayView* array_view,
|
|
2281
|
+
int64_t i);
|
|
2282
|
+
|
|
2283
|
+
/// \brief Get an element in an ArrowArrayView as an unsigned integer
|
|
2284
|
+
///
|
|
2285
|
+
/// This function does not check for null values, that values are actually integers, or
|
|
2286
|
+
/// that values are within a valid range for a uint64.
|
|
2287
|
+
static inline uint64_t ArrowArrayViewGetUIntUnsafe(
|
|
2288
|
+
const struct ArrowArrayView* array_view, int64_t i);
|
|
2289
|
+
|
|
2290
|
+
/// \brief Get an element in an ArrowArrayView as a double
|
|
2291
|
+
///
|
|
2292
|
+
/// This function does not check for null values, or
|
|
2293
|
+
/// that values are within a valid range for a double.
|
|
2294
|
+
static inline double ArrowArrayViewGetDoubleUnsafe(
|
|
2295
|
+
const struct ArrowArrayView* array_view, int64_t i);
|
|
2296
|
+
|
|
2297
|
+
/// \brief Get an element in an ArrowArrayView as an ArrowStringView
|
|
2298
|
+
///
|
|
2299
|
+
/// This function does not check for null values.
|
|
2300
|
+
static inline struct ArrowStringView ArrowArrayViewGetStringUnsafe(
|
|
2301
|
+
const struct ArrowArrayView* array_view, int64_t i);
|
|
2302
|
+
|
|
2303
|
+
/// \brief Get an element in an ArrowArrayView as an ArrowBufferView
|
|
2304
|
+
///
|
|
2305
|
+
/// This function does not check for null values.
|
|
2306
|
+
static inline struct ArrowBufferView ArrowArrayViewGetBytesUnsafe(
|
|
2307
|
+
const struct ArrowArrayView* array_view, int64_t i);
|
|
2308
|
+
|
|
2309
|
+
/// \brief Get an element in an ArrowArrayView as an ArrowDecimal
|
|
2310
|
+
///
|
|
2311
|
+
/// This function does not check for null values. The out parameter must
|
|
2312
|
+
/// be initialized with ArrowDecimalInit() with the proper parameters for this
|
|
2313
|
+
/// type before calling this for the first time.
|
|
2314
|
+
static inline void ArrowArrayViewGetDecimalUnsafe(const struct ArrowArrayView* array_view,
|
|
2315
|
+
int64_t i, struct ArrowDecimal* out);
|
|
2316
|
+
|
|
2317
|
+
/// @}
|
|
2318
|
+
|
|
2319
|
+
/// \defgroup nanoarrow-basic-array-stream Basic ArrowArrayStream implementation
|
|
2320
|
+
///
|
|
2321
|
+
/// An implementation of an ArrowArrayStream based on a collection of
|
|
2322
|
+
/// zero or more previously-existing ArrowArray objects. Users should
|
|
2323
|
+
/// initialize and/or validate the contents before transferring the
|
|
2324
|
+
/// responsibility of the ArrowArrayStream elsewhere.
|
|
2325
|
+
///
|
|
2326
|
+
/// @{
|
|
2327
|
+
|
|
2328
|
+
/// \brief Initialize an ArrowArrayStream backed by this implementation
|
|
2329
|
+
///
|
|
2330
|
+
/// This function moves the ownership of schema to the array_stream. If
|
|
2331
|
+
/// this function returns NANOARROW_OK, the caller is responsible for
|
|
2332
|
+
/// releasing the ArrowArrayStream.
|
|
2333
|
+
NANOARROW_DLL ArrowErrorCode ArrowBasicArrayStreamInit(
|
|
2334
|
+
struct ArrowArrayStream* array_stream, struct ArrowSchema* schema, int64_t n_arrays);
|
|
2335
|
+
|
|
2336
|
+
/// \brief Set the ith ArrowArray in this ArrowArrayStream.
|
|
2337
|
+
///
|
|
2338
|
+
/// array_stream must have been initialized with ArrowBasicArrayStreamInit().
|
|
2339
|
+
/// This function moves the ownership of array to the array_stream. i must
|
|
2340
|
+
/// be greater than or equal to zero and less than the value of n_arrays passed in
|
|
2341
|
+
/// ArrowBasicArrayStreamInit(). Callers are not required to fill all
|
|
2342
|
+
/// n_arrays members (i.e., n_arrays is a maximum bound).
|
|
2343
|
+
NANOARROW_DLL void ArrowBasicArrayStreamSetArray(struct ArrowArrayStream* array_stream,
|
|
2344
|
+
int64_t i, struct ArrowArray* array);
|
|
2345
|
+
|
|
2346
|
+
/// \brief Validate the contents of this ArrowArrayStream
|
|
2347
|
+
///
|
|
2348
|
+
/// array_stream must have been initialized with ArrowBasicArrayStreamInit().
|
|
2349
|
+
/// This function uses ArrowArrayStreamInitFromSchema() and ArrowArrayStreamSetArray()
|
|
2350
|
+
/// to validate the contents of the arrays.
|
|
2351
|
+
NANOARROW_DLL ArrowErrorCode ArrowBasicArrayStreamValidate(
|
|
2352
|
+
const struct ArrowArrayStream* array_stream, struct ArrowError* error);
|
|
2353
|
+
|
|
2354
|
+
/// @}
|
|
2355
|
+
|
|
2356
|
+
// Undefine ArrowErrorCode, which may have been defined to annotate functions that return
|
|
2357
|
+
// it to warn for an unused result.
|
|
2358
|
+
#if defined(ArrowErrorCode)
|
|
2359
|
+
#undef ArrowErrorCode
|
|
2360
|
+
#endif
|
|
2361
|
+
|
|
2362
|
+
// Inline function definitions
|
|
2363
|
+
|
|
2364
|
+
|
|
2365
|
+
|
|
2366
|
+
#ifdef __cplusplus
|
|
2367
|
+
}
|
|
2368
|
+
#endif
|
|
2369
|
+
|
|
2370
|
+
#endif
|
|
2371
|
+
// Licensed to the Apache Software Foundation (ASF) under one
|
|
2372
|
+
// or more contributor license agreements. See the NOTICE file
|
|
2373
|
+
// distributed with this work for additional information
|
|
2374
|
+
// regarding copyright ownership. The ASF licenses this file
|
|
2375
|
+
// to you under the Apache License, Version 2.0 (the
|
|
2376
|
+
// "License"); you may not use this file except in compliance
|
|
2377
|
+
// with the License. You may obtain a copy of the License at
|
|
2378
|
+
//
|
|
2379
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
2380
|
+
//
|
|
2381
|
+
// Unless required by applicable law or agreed to in writing,
|
|
2382
|
+
// software distributed under the License is distributed on an
|
|
2383
|
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
2384
|
+
// KIND, either express or implied. See the License for the
|
|
2385
|
+
// specific language governing permissions and limitations
|
|
2386
|
+
// under the License.
|
|
2387
|
+
|
|
2388
|
+
#ifndef NANOARROW_BUFFER_INLINE_H_INCLUDED
|
|
2389
|
+
#define NANOARROW_BUFFER_INLINE_H_INCLUDED
|
|
2390
|
+
|
|
2391
|
+
#include <errno.h>
|
|
2392
|
+
#include <stdint.h>
|
|
2393
|
+
#include <string.h>
|
|
2394
|
+
|
|
2395
|
+
|
|
2396
|
+
|
|
2397
|
+
#ifdef __cplusplus
|
|
2398
|
+
extern "C" {
|
|
2399
|
+
#endif
|
|
2400
|
+
|
|
2401
|
+
// Modified from Arrow C++ (1eb46f76) cpp/src/arrow/chunk_resolver.h#L133-L162
|
|
2402
|
+
static inline int64_t ArrowResolveChunk64(int64_t index, const int64_t* offsets,
|
|
2403
|
+
int64_t lo, int64_t hi) {
|
|
2404
|
+
// Similar to std::upper_bound(), but slightly different as our offsets
|
|
2405
|
+
// array always starts with 0.
|
|
2406
|
+
int64_t n = hi - lo;
|
|
2407
|
+
// First iteration does not need to check for n > 1
|
|
2408
|
+
// (lo < hi is guaranteed by the precondition).
|
|
2409
|
+
NANOARROW_DCHECK(n > 1);
|
|
2410
|
+
do {
|
|
2411
|
+
const int64_t m = n >> 1;
|
|
2412
|
+
const int64_t mid = lo + m;
|
|
2413
|
+
if (index >= offsets[mid]) {
|
|
2414
|
+
lo = mid;
|
|
2415
|
+
n -= m;
|
|
2416
|
+
} else {
|
|
2417
|
+
n = m;
|
|
2418
|
+
}
|
|
2419
|
+
} while (n > 1);
|
|
2420
|
+
return lo;
|
|
2421
|
+
}
|
|
2422
|
+
|
|
2423
|
+
static inline int64_t ArrowResolveChunk32(int32_t index, const int32_t* offsets,
|
|
2424
|
+
int32_t lo, int32_t hi) {
|
|
2425
|
+
// Similar to std::upper_bound(), but slightly different as our offsets
|
|
2426
|
+
// array always starts with 0.
|
|
2427
|
+
int32_t n = hi - lo;
|
|
2428
|
+
// First iteration does not need to check for n > 1
|
|
2429
|
+
// (lo < hi is guaranteed by the precondition).
|
|
2430
|
+
NANOARROW_DCHECK(n > 1);
|
|
2431
|
+
do {
|
|
2432
|
+
const int32_t m = n >> 1;
|
|
2433
|
+
const int32_t mid = lo + m;
|
|
2434
|
+
if (index >= offsets[mid]) {
|
|
2435
|
+
lo = mid;
|
|
2436
|
+
n -= m;
|
|
2437
|
+
} else {
|
|
2438
|
+
n = m;
|
|
2439
|
+
}
|
|
2440
|
+
} while (n > 1);
|
|
2441
|
+
return lo;
|
|
2442
|
+
}
|
|
2443
|
+
|
|
2444
|
+
static inline int64_t _ArrowGrowByFactor(int64_t current_capacity, int64_t new_capacity) {
|
|
2445
|
+
int64_t doubled_capacity = current_capacity * 2;
|
|
2446
|
+
if (doubled_capacity > new_capacity) {
|
|
2447
|
+
return doubled_capacity;
|
|
2448
|
+
} else {
|
|
2449
|
+
return new_capacity;
|
|
2450
|
+
}
|
|
2451
|
+
}
|
|
2452
|
+
|
|
2453
|
+
// float to half float conversion, adapted from Arrow Go
|
|
2454
|
+
// https://github.com/apache/arrow/blob/main/go/arrow/float16/float16.go
|
|
2455
|
+
static inline uint16_t ArrowFloatToHalfFloat(float value) {
|
|
2456
|
+
union {
|
|
2457
|
+
float f;
|
|
2458
|
+
uint32_t b;
|
|
2459
|
+
} u;
|
|
2460
|
+
u.f = value;
|
|
2461
|
+
|
|
2462
|
+
uint16_t sn = (uint16_t)((u.b >> 31) & 0x1);
|
|
2463
|
+
uint16_t exp = (u.b >> 23) & 0xff;
|
|
2464
|
+
int16_t res = (int16_t)(exp - 127 + 15);
|
|
2465
|
+
uint16_t fc = (uint16_t)(u.b >> 13) & 0x3ff;
|
|
2466
|
+
|
|
2467
|
+
if (exp == 0) {
|
|
2468
|
+
res = 0;
|
|
2469
|
+
} else if (exp == 0xff) {
|
|
2470
|
+
res = 0x1f;
|
|
2471
|
+
} else if (res > 0x1e) {
|
|
2472
|
+
res = 0x1f;
|
|
2473
|
+
fc = 0;
|
|
2474
|
+
} else if (res < 0x01) {
|
|
2475
|
+
res = 0;
|
|
2476
|
+
fc = 0;
|
|
2477
|
+
}
|
|
2478
|
+
|
|
2479
|
+
return (uint16_t)((sn << 15) | (uint16_t)(res << 10) | fc);
|
|
2480
|
+
}
|
|
2481
|
+
|
|
2482
|
+
// half float to float conversion, adapted from Arrow Go
|
|
2483
|
+
// https://github.com/apache/arrow/blob/main/go/arrow/float16/float16.go
|
|
2484
|
+
static inline float ArrowHalfFloatToFloat(uint16_t value) {
|
|
2485
|
+
uint32_t sn = (uint32_t)((value >> 15) & 0x1);
|
|
2486
|
+
uint32_t exp = (value >> 10) & 0x1f;
|
|
2487
|
+
uint32_t res = exp + 127 - 15;
|
|
2488
|
+
uint32_t fc = value & 0x3ff;
|
|
2489
|
+
|
|
2490
|
+
if (exp == 0) {
|
|
2491
|
+
res = 0;
|
|
2492
|
+
} else if (exp == 0x1f) {
|
|
2493
|
+
res = 0xff;
|
|
2494
|
+
}
|
|
2495
|
+
|
|
2496
|
+
union {
|
|
2497
|
+
float f;
|
|
2498
|
+
uint32_t b;
|
|
2499
|
+
} u;
|
|
2500
|
+
u.b = (uint32_t)(sn << 31) | (uint32_t)(res << 23) | (uint32_t)(fc << 13);
|
|
2501
|
+
return u.f;
|
|
2502
|
+
}
|
|
2503
|
+
|
|
2504
|
+
static inline void ArrowBufferInit(struct ArrowBuffer* buffer) {
|
|
2505
|
+
buffer->data = NULL;
|
|
2506
|
+
buffer->size_bytes = 0;
|
|
2507
|
+
buffer->capacity_bytes = 0;
|
|
2508
|
+
buffer->allocator = ArrowBufferAllocatorDefault();
|
|
2509
|
+
}
|
|
2510
|
+
|
|
2511
|
+
static inline ArrowErrorCode ArrowBufferSetAllocator(
|
|
2512
|
+
struct ArrowBuffer* buffer, struct ArrowBufferAllocator allocator) {
|
|
2513
|
+
// This is not a perfect test for "has a buffer already been allocated"
|
|
2514
|
+
// but is likely to catch most cases.
|
|
2515
|
+
if (buffer->data == NULL) {
|
|
2516
|
+
buffer->allocator = allocator;
|
|
2517
|
+
return NANOARROW_OK;
|
|
2518
|
+
} else {
|
|
2519
|
+
return EINVAL;
|
|
2520
|
+
}
|
|
2521
|
+
}
|
|
2522
|
+
|
|
2523
|
+
static inline void ArrowBufferReset(struct ArrowBuffer* buffer) {
|
|
2524
|
+
buffer->allocator.free(&buffer->allocator, (uint8_t*)buffer->data,
|
|
2525
|
+
buffer->capacity_bytes);
|
|
2526
|
+
ArrowBufferInit(buffer);
|
|
2527
|
+
}
|
|
2528
|
+
|
|
2529
|
+
static inline void ArrowBufferMove(struct ArrowBuffer* src, struct ArrowBuffer* dst) {
|
|
2530
|
+
memcpy(dst, src, sizeof(struct ArrowBuffer));
|
|
2531
|
+
src->data = NULL;
|
|
2532
|
+
ArrowBufferInit(src);
|
|
2533
|
+
}
|
|
2534
|
+
|
|
2535
|
+
static inline ArrowErrorCode ArrowBufferResize(struct ArrowBuffer* buffer,
|
|
2536
|
+
int64_t new_size_bytes,
|
|
2537
|
+
char shrink_to_fit) {
|
|
2538
|
+
if (new_size_bytes < 0) {
|
|
2539
|
+
return EINVAL;
|
|
2540
|
+
}
|
|
2541
|
+
|
|
2542
|
+
int needs_reallocation = new_size_bytes > buffer->capacity_bytes ||
|
|
2543
|
+
(shrink_to_fit && new_size_bytes < buffer->capacity_bytes);
|
|
2544
|
+
|
|
2545
|
+
if (needs_reallocation) {
|
|
2546
|
+
buffer->data = buffer->allocator.reallocate(&buffer->allocator, buffer->data,
|
|
2547
|
+
buffer->capacity_bytes, new_size_bytes);
|
|
2548
|
+
|
|
2549
|
+
if (buffer->data == NULL && new_size_bytes > 0) {
|
|
2550
|
+
buffer->capacity_bytes = 0;
|
|
2551
|
+
buffer->size_bytes = 0;
|
|
2552
|
+
return ENOMEM;
|
|
2553
|
+
}
|
|
2554
|
+
|
|
2555
|
+
buffer->capacity_bytes = new_size_bytes;
|
|
2556
|
+
}
|
|
2557
|
+
|
|
2558
|
+
buffer->size_bytes = new_size_bytes;
|
|
2559
|
+
return NANOARROW_OK;
|
|
2560
|
+
}
|
|
2561
|
+
|
|
2562
|
+
static inline ArrowErrorCode ArrowBufferReserve(struct ArrowBuffer* buffer,
|
|
2563
|
+
int64_t additional_size_bytes) {
|
|
2564
|
+
int64_t min_capacity_bytes = buffer->size_bytes + additional_size_bytes;
|
|
2565
|
+
if (min_capacity_bytes <= buffer->capacity_bytes) {
|
|
2566
|
+
return NANOARROW_OK;
|
|
2567
|
+
}
|
|
2568
|
+
|
|
2569
|
+
int64_t new_capacity_bytes =
|
|
2570
|
+
_ArrowGrowByFactor(buffer->capacity_bytes, min_capacity_bytes);
|
|
2571
|
+
buffer->data = buffer->allocator.reallocate(&buffer->allocator, buffer->data,
|
|
2572
|
+
buffer->capacity_bytes, new_capacity_bytes);
|
|
2573
|
+
|
|
2574
|
+
if (buffer->data == NULL && new_capacity_bytes > 0) {
|
|
2575
|
+
buffer->capacity_bytes = 0;
|
|
2576
|
+
buffer->size_bytes = 0;
|
|
2577
|
+
return ENOMEM;
|
|
2578
|
+
}
|
|
2579
|
+
|
|
2580
|
+
buffer->capacity_bytes = new_capacity_bytes;
|
|
2581
|
+
return NANOARROW_OK;
|
|
2582
|
+
}
|
|
2583
|
+
|
|
2584
|
+
static inline void ArrowBufferAppendUnsafe(struct ArrowBuffer* buffer, const void* data,
|
|
2585
|
+
int64_t size_bytes) {
|
|
2586
|
+
if (size_bytes > 0) {
|
|
2587
|
+
NANOARROW_DCHECK(buffer->data != NULL);
|
|
2588
|
+
memcpy(buffer->data + buffer->size_bytes, data, size_bytes);
|
|
2589
|
+
buffer->size_bytes += size_bytes;
|
|
2590
|
+
}
|
|
2591
|
+
}
|
|
2592
|
+
|
|
2593
|
+
static inline ArrowErrorCode ArrowBufferAppend(struct ArrowBuffer* buffer,
|
|
2594
|
+
const void* data, int64_t size_bytes) {
|
|
2595
|
+
NANOARROW_RETURN_NOT_OK(ArrowBufferReserve(buffer, size_bytes));
|
|
2596
|
+
|
|
2597
|
+
ArrowBufferAppendUnsafe(buffer, data, size_bytes);
|
|
2598
|
+
return NANOARROW_OK;
|
|
2599
|
+
}
|
|
2600
|
+
|
|
2601
|
+
static inline ArrowErrorCode ArrowBufferAppendInt8(struct ArrowBuffer* buffer,
|
|
2602
|
+
int8_t value) {
|
|
2603
|
+
return ArrowBufferAppend(buffer, &value, sizeof(int8_t));
|
|
2604
|
+
}
|
|
2605
|
+
|
|
2606
|
+
static inline ArrowErrorCode ArrowBufferAppendUInt8(struct ArrowBuffer* buffer,
|
|
2607
|
+
uint8_t value) {
|
|
2608
|
+
return ArrowBufferAppend(buffer, &value, sizeof(uint8_t));
|
|
2609
|
+
}
|
|
2610
|
+
|
|
2611
|
+
static inline ArrowErrorCode ArrowBufferAppendInt16(struct ArrowBuffer* buffer,
|
|
2612
|
+
int16_t value) {
|
|
2613
|
+
return ArrowBufferAppend(buffer, &value, sizeof(int16_t));
|
|
2614
|
+
}
|
|
2615
|
+
|
|
2616
|
+
static inline ArrowErrorCode ArrowBufferAppendUInt16(struct ArrowBuffer* buffer,
|
|
2617
|
+
uint16_t value) {
|
|
2618
|
+
return ArrowBufferAppend(buffer, &value, sizeof(uint16_t));
|
|
2619
|
+
}
|
|
2620
|
+
|
|
2621
|
+
static inline ArrowErrorCode ArrowBufferAppendInt32(struct ArrowBuffer* buffer,
|
|
2622
|
+
int32_t value) {
|
|
2623
|
+
return ArrowBufferAppend(buffer, &value, sizeof(int32_t));
|
|
2624
|
+
}
|
|
2625
|
+
|
|
2626
|
+
static inline ArrowErrorCode ArrowBufferAppendUInt32(struct ArrowBuffer* buffer,
|
|
2627
|
+
uint32_t value) {
|
|
2628
|
+
return ArrowBufferAppend(buffer, &value, sizeof(uint32_t));
|
|
2629
|
+
}
|
|
2630
|
+
|
|
2631
|
+
static inline ArrowErrorCode ArrowBufferAppendInt64(struct ArrowBuffer* buffer,
|
|
2632
|
+
int64_t value) {
|
|
2633
|
+
return ArrowBufferAppend(buffer, &value, sizeof(int64_t));
|
|
2634
|
+
}
|
|
2635
|
+
|
|
2636
|
+
static inline ArrowErrorCode ArrowBufferAppendUInt64(struct ArrowBuffer* buffer,
|
|
2637
|
+
uint64_t value) {
|
|
2638
|
+
return ArrowBufferAppend(buffer, &value, sizeof(uint64_t));
|
|
2639
|
+
}
|
|
2640
|
+
|
|
2641
|
+
static inline ArrowErrorCode ArrowBufferAppendDouble(struct ArrowBuffer* buffer,
|
|
2642
|
+
double value) {
|
|
2643
|
+
return ArrowBufferAppend(buffer, &value, sizeof(double));
|
|
2644
|
+
}
|
|
2645
|
+
|
|
2646
|
+
static inline ArrowErrorCode ArrowBufferAppendFloat(struct ArrowBuffer* buffer,
|
|
2647
|
+
float value) {
|
|
2648
|
+
return ArrowBufferAppend(buffer, &value, sizeof(float));
|
|
2649
|
+
}
|
|
2650
|
+
|
|
2651
|
+
static inline ArrowErrorCode ArrowBufferAppendStringView(struct ArrowBuffer* buffer,
|
|
2652
|
+
struct ArrowStringView value) {
|
|
2653
|
+
return ArrowBufferAppend(buffer, value.data, value.size_bytes);
|
|
2654
|
+
}
|
|
2655
|
+
|
|
2656
|
+
static inline ArrowErrorCode ArrowBufferAppendBufferView(struct ArrowBuffer* buffer,
|
|
2657
|
+
struct ArrowBufferView value) {
|
|
2658
|
+
return ArrowBufferAppend(buffer, value.data.data, value.size_bytes);
|
|
2659
|
+
}
|
|
2660
|
+
|
|
2661
|
+
static inline ArrowErrorCode ArrowBufferAppendFill(struct ArrowBuffer* buffer,
|
|
2662
|
+
uint8_t value, int64_t size_bytes) {
|
|
2663
|
+
if (size_bytes == 0) {
|
|
2664
|
+
return NANOARROW_OK;
|
|
2665
|
+
}
|
|
2666
|
+
|
|
2667
|
+
NANOARROW_RETURN_NOT_OK(ArrowBufferReserve(buffer, size_bytes));
|
|
2668
|
+
|
|
2669
|
+
NANOARROW_DCHECK(buffer->data != NULL); // To help clang-tidy
|
|
2670
|
+
memset(buffer->data + buffer->size_bytes, value, size_bytes);
|
|
2671
|
+
buffer->size_bytes += size_bytes;
|
|
2672
|
+
|
|
2673
|
+
return NANOARROW_OK;
|
|
2674
|
+
}
|
|
2675
|
+
|
|
2676
|
+
static const uint8_t _ArrowkBitmask[] = {1, 2, 4, 8, 16, 32, 64, 128};
|
|
2677
|
+
static const uint8_t _ArrowkFlippedBitmask[] = {254, 253, 251, 247, 239, 223, 191, 127};
|
|
2678
|
+
static const uint8_t _ArrowkPrecedingBitmask[] = {0, 1, 3, 7, 15, 31, 63, 127};
|
|
2679
|
+
static const uint8_t _ArrowkTrailingBitmask[] = {255, 254, 252, 248, 240, 224, 192, 128};
|
|
2680
|
+
|
|
2681
|
+
static const uint8_t _ArrowkBytePopcount[] = {
|
|
2682
|
+
0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3,
|
|
2683
|
+
4, 4, 5, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4,
|
|
2684
|
+
4, 5, 4, 5, 5, 6, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4,
|
|
2685
|
+
5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5,
|
|
2686
|
+
4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2,
|
|
2687
|
+
3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5,
|
|
2688
|
+
5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4,
|
|
2689
|
+
5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 3, 4, 4, 5, 4, 5, 5, 6,
|
|
2690
|
+
4, 5, 5, 6, 5, 6, 6, 7, 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8};
|
|
2691
|
+
|
|
2692
|
+
static inline int64_t _ArrowRoundUpToMultipleOf8(int64_t value) {
|
|
2693
|
+
return (value + 7) & ~((int64_t)7);
|
|
2694
|
+
}
|
|
2695
|
+
|
|
2696
|
+
static inline int64_t _ArrowRoundDownToMultipleOf8(int64_t value) {
|
|
2697
|
+
return (value / 8) * 8;
|
|
2698
|
+
}
|
|
2699
|
+
|
|
2700
|
+
static inline int64_t _ArrowBytesForBits(int64_t bits) {
|
|
2701
|
+
return (bits >> 3) + ((bits & 7) != 0);
|
|
2702
|
+
}
|
|
2703
|
+
|
|
2704
|
+
static inline void _ArrowBitsUnpackInt8(const uint8_t word, int8_t* out) {
|
|
2705
|
+
out[0] = (word & 0x1) != 0;
|
|
2706
|
+
out[1] = (word & 0x2) != 0;
|
|
2707
|
+
out[2] = (word & 0x4) != 0;
|
|
2708
|
+
out[3] = (word & 0x8) != 0;
|
|
2709
|
+
out[4] = (word & 0x10) != 0;
|
|
2710
|
+
out[5] = (word & 0x20) != 0;
|
|
2711
|
+
out[6] = (word & 0x40) != 0;
|
|
2712
|
+
out[7] = (word & 0x80) != 0;
|
|
2713
|
+
}
|
|
2714
|
+
|
|
2715
|
+
static inline void _ArrowBitsUnpackInt32(const uint8_t word, int32_t* out) {
|
|
2716
|
+
out[0] = (word & 0x1) != 0;
|
|
2717
|
+
out[1] = (word & 0x2) != 0;
|
|
2718
|
+
out[2] = (word & 0x4) != 0;
|
|
2719
|
+
out[3] = (word & 0x8) != 0;
|
|
2720
|
+
out[4] = (word & 0x10) != 0;
|
|
2721
|
+
out[5] = (word & 0x20) != 0;
|
|
2722
|
+
out[6] = (word & 0x40) != 0;
|
|
2723
|
+
out[7] = (word & 0x80) != 0;
|
|
2724
|
+
}
|
|
2725
|
+
|
|
2726
|
+
static inline void _ArrowBitmapPackInt8(const int8_t* values, uint8_t* out) {
|
|
2727
|
+
*out = (uint8_t)(values[0] | ((values[1] + 0x1) & 0x2) | ((values[2] + 0x3) & 0x4) |
|
|
2728
|
+
((values[3] + 0x7) & 0x8) | ((values[4] + 0xf) & 0x10) |
|
|
2729
|
+
((values[5] + 0x1f) & 0x20) | ((values[6] + 0x3f) & 0x40) |
|
|
2730
|
+
((values[7] + 0x7f) & 0x80));
|
|
2731
|
+
}
|
|
2732
|
+
|
|
2733
|
+
static inline void _ArrowBitmapPackInt32(const int32_t* values, uint8_t* out) {
|
|
2734
|
+
*out = (uint8_t)(values[0] | ((values[1] + 0x1) & 0x2) | ((values[2] + 0x3) & 0x4) |
|
|
2735
|
+
((values[3] + 0x7) & 0x8) | ((values[4] + 0xf) & 0x10) |
|
|
2736
|
+
((values[5] + 0x1f) & 0x20) | ((values[6] + 0x3f) & 0x40) |
|
|
2737
|
+
((values[7] + 0x7f) & 0x80));
|
|
2738
|
+
}
|
|
2739
|
+
|
|
2740
|
+
static inline int8_t ArrowBitGet(const uint8_t* bits, int64_t i) {
|
|
2741
|
+
return (bits[i >> 3] >> (i & 0x07)) & 1;
|
|
2742
|
+
}
|
|
2743
|
+
|
|
2744
|
+
static inline void ArrowBitsUnpackInt8(const uint8_t* bits, int64_t start_offset,
|
|
2745
|
+
int64_t length, int8_t* out) {
|
|
2746
|
+
if (length == 0) {
|
|
2747
|
+
return;
|
|
2748
|
+
}
|
|
2749
|
+
|
|
2750
|
+
const int64_t i_begin = start_offset;
|
|
2751
|
+
const int64_t i_end = start_offset + length;
|
|
2752
|
+
const int64_t i_last_valid = i_end - 1;
|
|
2753
|
+
|
|
2754
|
+
const int64_t bytes_begin = i_begin / 8;
|
|
2755
|
+
const int64_t bytes_last_valid = i_last_valid / 8;
|
|
2756
|
+
|
|
2757
|
+
if (bytes_begin == bytes_last_valid) {
|
|
2758
|
+
for (int i = 0; i < length; i++) {
|
|
2759
|
+
out[i] = ArrowBitGet(&bits[bytes_begin], i + i_begin % 8);
|
|
2760
|
+
}
|
|
2761
|
+
|
|
2762
|
+
return;
|
|
2763
|
+
}
|
|
2764
|
+
|
|
2765
|
+
// first byte
|
|
2766
|
+
for (int i = 0; i < 8 - (i_begin % 8); i++) {
|
|
2767
|
+
*out++ = ArrowBitGet(&bits[bytes_begin], i + i_begin % 8);
|
|
2768
|
+
}
|
|
2769
|
+
|
|
2770
|
+
// middle bytes
|
|
2771
|
+
for (int64_t i = bytes_begin + 1; i < bytes_last_valid; i++) {
|
|
2772
|
+
_ArrowBitsUnpackInt8(bits[i], out);
|
|
2773
|
+
out += 8;
|
|
2774
|
+
}
|
|
2775
|
+
|
|
2776
|
+
// last byte
|
|
2777
|
+
const int bits_remaining = (int)(i_end % 8 == 0 ? 8 : i_end % 8);
|
|
2778
|
+
for (int i = 0; i < bits_remaining; i++) {
|
|
2779
|
+
*out++ = ArrowBitGet(&bits[bytes_last_valid], i);
|
|
2780
|
+
}
|
|
2781
|
+
}
|
|
2782
|
+
|
|
2783
|
+
static inline void ArrowBitsUnpackInt32(const uint8_t* bits, int64_t start_offset,
|
|
2784
|
+
int64_t length, int32_t* out) {
|
|
2785
|
+
if (length == 0) {
|
|
2786
|
+
return;
|
|
2787
|
+
}
|
|
2788
|
+
|
|
2789
|
+
NANOARROW_DCHECK(bits != NULL && out != NULL);
|
|
2790
|
+
|
|
2791
|
+
const int64_t i_begin = start_offset;
|
|
2792
|
+
const int64_t i_end = start_offset + length;
|
|
2793
|
+
const int64_t i_last_valid = i_end - 1;
|
|
2794
|
+
|
|
2795
|
+
const int64_t bytes_begin = i_begin / 8;
|
|
2796
|
+
const int64_t bytes_last_valid = i_last_valid / 8;
|
|
2797
|
+
|
|
2798
|
+
if (bytes_begin == bytes_last_valid) {
|
|
2799
|
+
for (int i = 0; i < length; i++) {
|
|
2800
|
+
out[i] = ArrowBitGet(&bits[bytes_begin], i + i_begin % 8);
|
|
2801
|
+
}
|
|
2802
|
+
|
|
2803
|
+
return;
|
|
2804
|
+
}
|
|
2805
|
+
|
|
2806
|
+
// first byte
|
|
2807
|
+
for (int i = 0; i < 8 - (i_begin % 8); i++) {
|
|
2808
|
+
*out++ = ArrowBitGet(&bits[bytes_begin], i + i_begin % 8);
|
|
2809
|
+
}
|
|
2810
|
+
|
|
2811
|
+
// middle bytes
|
|
2812
|
+
for (int64_t i = bytes_begin + 1; i < bytes_last_valid; i++) {
|
|
2813
|
+
_ArrowBitsUnpackInt32(bits[i], out);
|
|
2814
|
+
out += 8;
|
|
2815
|
+
}
|
|
2816
|
+
|
|
2817
|
+
// last byte
|
|
2818
|
+
const int bits_remaining = (int)(i_end % 8 == 0 ? 8 : i_end % 8);
|
|
2819
|
+
for (int i = 0; i < bits_remaining; i++) {
|
|
2820
|
+
*out++ = ArrowBitGet(&bits[bytes_last_valid], i);
|
|
2821
|
+
}
|
|
2822
|
+
}
|
|
2823
|
+
|
|
2824
|
+
static inline void ArrowBitSet(uint8_t* bits, int64_t i) {
|
|
2825
|
+
bits[i / 8] |= _ArrowkBitmask[i % 8];
|
|
2826
|
+
}
|
|
2827
|
+
|
|
2828
|
+
static inline void ArrowBitClear(uint8_t* bits, int64_t i) {
|
|
2829
|
+
bits[i / 8] &= _ArrowkFlippedBitmask[i % 8];
|
|
2830
|
+
}
|
|
2831
|
+
|
|
2832
|
+
static inline void ArrowBitSetTo(uint8_t* bits, int64_t i, uint8_t bit_is_set) {
|
|
2833
|
+
bits[i / 8] ^= (uint8_t)(((uint8_t)(-((uint8_t)(bit_is_set != 0)) ^ bits[i / 8])) &
|
|
2834
|
+
_ArrowkBitmask[i % 8]);
|
|
2835
|
+
}
|
|
2836
|
+
|
|
2837
|
+
static inline void ArrowBitsSetTo(uint8_t* bits, int64_t start_offset, int64_t length,
|
|
2838
|
+
uint8_t bits_are_set) {
|
|
2839
|
+
if (length == 0) {
|
|
2840
|
+
return;
|
|
2841
|
+
}
|
|
2842
|
+
|
|
2843
|
+
NANOARROW_DCHECK(bits != NULL);
|
|
2844
|
+
|
|
2845
|
+
const int64_t i_begin = start_offset;
|
|
2846
|
+
const int64_t i_end = start_offset + length;
|
|
2847
|
+
const uint8_t fill_byte = (uint8_t)(-bits_are_set);
|
|
2848
|
+
|
|
2849
|
+
const int64_t bytes_begin = i_begin / 8;
|
|
2850
|
+
const int64_t bytes_end = i_end / 8 + 1;
|
|
2851
|
+
|
|
2852
|
+
const uint8_t first_byte_mask = _ArrowkPrecedingBitmask[i_begin % 8];
|
|
2853
|
+
const uint8_t last_byte_mask = _ArrowkTrailingBitmask[i_end % 8];
|
|
2854
|
+
|
|
2855
|
+
if (bytes_end == bytes_begin + 1) {
|
|
2856
|
+
// set bits within a single byte
|
|
2857
|
+
const uint8_t only_byte_mask =
|
|
2858
|
+
i_end % 8 == 0 ? first_byte_mask : (uint8_t)(first_byte_mask | last_byte_mask);
|
|
2859
|
+
bits[bytes_begin] &= only_byte_mask;
|
|
2860
|
+
bits[bytes_begin] |= (uint8_t)(fill_byte & ~only_byte_mask);
|
|
2861
|
+
return;
|
|
2862
|
+
}
|
|
2863
|
+
|
|
2864
|
+
// set/clear trailing bits of first byte
|
|
2865
|
+
bits[bytes_begin] &= first_byte_mask;
|
|
2866
|
+
bits[bytes_begin] |= (uint8_t)(fill_byte & ~first_byte_mask);
|
|
2867
|
+
|
|
2868
|
+
if (bytes_end - bytes_begin > 2) {
|
|
2869
|
+
// set/clear whole bytes
|
|
2870
|
+
memset(bits + bytes_begin + 1, fill_byte, (size_t)(bytes_end - bytes_begin - 2));
|
|
2871
|
+
}
|
|
2872
|
+
|
|
2873
|
+
if (i_end % 8 == 0) {
|
|
2874
|
+
return;
|
|
2875
|
+
}
|
|
2876
|
+
|
|
2877
|
+
// set/clear leading bits of last byte
|
|
2878
|
+
bits[bytes_end - 1] &= last_byte_mask;
|
|
2879
|
+
bits[bytes_end - 1] |= (uint8_t)(fill_byte & ~last_byte_mask);
|
|
2880
|
+
}
|
|
2881
|
+
|
|
2882
|
+
static inline int64_t ArrowBitCountSet(const uint8_t* bits, int64_t start_offset,
|
|
2883
|
+
int64_t length) {
|
|
2884
|
+
if (length == 0) {
|
|
2885
|
+
return 0;
|
|
2886
|
+
}
|
|
2887
|
+
|
|
2888
|
+
NANOARROW_DCHECK(bits != NULL);
|
|
2889
|
+
|
|
2890
|
+
const int64_t i_begin = start_offset;
|
|
2891
|
+
const int64_t i_end = start_offset + length;
|
|
2892
|
+
const int64_t i_last_valid = i_end - 1;
|
|
2893
|
+
|
|
2894
|
+
const int64_t bytes_begin = i_begin / 8;
|
|
2895
|
+
const int64_t bytes_last_valid = i_last_valid / 8;
|
|
2896
|
+
|
|
2897
|
+
if (bytes_begin == bytes_last_valid) {
|
|
2898
|
+
// count bits within a single byte
|
|
2899
|
+
const uint8_t first_byte_mask = _ArrowkPrecedingBitmask[i_end % 8];
|
|
2900
|
+
const uint8_t last_byte_mask = _ArrowkTrailingBitmask[i_begin % 8];
|
|
2901
|
+
|
|
2902
|
+
const uint8_t only_byte_mask =
|
|
2903
|
+
i_end % 8 == 0 ? last_byte_mask : (uint8_t)(first_byte_mask & last_byte_mask);
|
|
2904
|
+
|
|
2905
|
+
const uint8_t byte_masked = bits[bytes_begin] & only_byte_mask;
|
|
2906
|
+
return _ArrowkBytePopcount[byte_masked];
|
|
2907
|
+
}
|
|
2908
|
+
|
|
2909
|
+
const uint8_t first_byte_mask = _ArrowkPrecedingBitmask[i_begin % 8];
|
|
2910
|
+
const uint8_t last_byte_mask = i_end % 8 == 0 ? 0 : _ArrowkTrailingBitmask[i_end % 8];
|
|
2911
|
+
int64_t count = 0;
|
|
2912
|
+
|
|
2913
|
+
// first byte
|
|
2914
|
+
count += _ArrowkBytePopcount[bits[bytes_begin] & ~first_byte_mask];
|
|
2915
|
+
|
|
2916
|
+
// middle bytes
|
|
2917
|
+
for (int64_t i = bytes_begin + 1; i < bytes_last_valid; i++) {
|
|
2918
|
+
count += _ArrowkBytePopcount[bits[i]];
|
|
2919
|
+
}
|
|
2920
|
+
|
|
2921
|
+
// last byte
|
|
2922
|
+
count += _ArrowkBytePopcount[bits[bytes_last_valid] & ~last_byte_mask];
|
|
2923
|
+
|
|
2924
|
+
return count;
|
|
2925
|
+
}
|
|
2926
|
+
|
|
2927
|
+
static inline void ArrowBitmapInit(struct ArrowBitmap* bitmap) {
|
|
2928
|
+
ArrowBufferInit(&bitmap->buffer);
|
|
2929
|
+
bitmap->size_bits = 0;
|
|
2930
|
+
}
|
|
2931
|
+
|
|
2932
|
+
static inline void ArrowBitmapMove(struct ArrowBitmap* src, struct ArrowBitmap* dst) {
|
|
2933
|
+
ArrowBufferMove(&src->buffer, &dst->buffer);
|
|
2934
|
+
dst->size_bits = src->size_bits;
|
|
2935
|
+
src->size_bits = 0;
|
|
2936
|
+
}
|
|
2937
|
+
|
|
2938
|
+
static inline ArrowErrorCode ArrowBitmapReserve(struct ArrowBitmap* bitmap,
|
|
2939
|
+
int64_t additional_size_bits) {
|
|
2940
|
+
int64_t min_capacity_bits = bitmap->size_bits + additional_size_bits;
|
|
2941
|
+
int64_t min_capacity_bytes = _ArrowBytesForBits(min_capacity_bits);
|
|
2942
|
+
int64_t current_size_bytes = bitmap->buffer.size_bytes;
|
|
2943
|
+
int64_t current_capacity_bytes = bitmap->buffer.capacity_bytes;
|
|
2944
|
+
|
|
2945
|
+
if (min_capacity_bytes <= current_capacity_bytes) {
|
|
2946
|
+
return NANOARROW_OK;
|
|
2947
|
+
}
|
|
2948
|
+
|
|
2949
|
+
int64_t additional_capacity_bytes = min_capacity_bytes - current_size_bytes;
|
|
2950
|
+
NANOARROW_RETURN_NOT_OK(ArrowBufferReserve(&bitmap->buffer, additional_capacity_bytes));
|
|
2951
|
+
|
|
2952
|
+
// Zero out the last byte for deterministic output in the common case
|
|
2953
|
+
// of reserving a known remaining size. We should have returned above
|
|
2954
|
+
// if there was not at least one additional byte to allocate; however,
|
|
2955
|
+
// DCHECK() just to be sure.
|
|
2956
|
+
NANOARROW_DCHECK(bitmap->buffer.capacity_bytes > current_capacity_bytes);
|
|
2957
|
+
bitmap->buffer.data[bitmap->buffer.capacity_bytes - 1] = 0;
|
|
2958
|
+
return NANOARROW_OK;
|
|
2959
|
+
}
|
|
2960
|
+
|
|
2961
|
+
static inline ArrowErrorCode ArrowBitmapResize(struct ArrowBitmap* bitmap,
|
|
2962
|
+
int64_t new_size_bits,
|
|
2963
|
+
char shrink_to_fit) {
|
|
2964
|
+
if (new_size_bits < 0) {
|
|
2965
|
+
return EINVAL;
|
|
2966
|
+
}
|
|
2967
|
+
|
|
2968
|
+
int64_t new_size_bytes = _ArrowBytesForBits(new_size_bits);
|
|
2969
|
+
NANOARROW_RETURN_NOT_OK(
|
|
2970
|
+
ArrowBufferResize(&bitmap->buffer, new_size_bytes, shrink_to_fit));
|
|
2971
|
+
|
|
2972
|
+
bitmap->size_bits = new_size_bits;
|
|
2973
|
+
return NANOARROW_OK;
|
|
2974
|
+
}
|
|
2975
|
+
|
|
2976
|
+
static inline ArrowErrorCode ArrowBitmapAppend(struct ArrowBitmap* bitmap,
|
|
2977
|
+
uint8_t bits_are_set, int64_t length) {
|
|
2978
|
+
NANOARROW_RETURN_NOT_OK(ArrowBitmapReserve(bitmap, length));
|
|
2979
|
+
|
|
2980
|
+
ArrowBitmapAppendUnsafe(bitmap, bits_are_set, length);
|
|
2981
|
+
return NANOARROW_OK;
|
|
2982
|
+
}
|
|
2983
|
+
|
|
2984
|
+
static inline void ArrowBitmapAppendUnsafe(struct ArrowBitmap* bitmap,
|
|
2985
|
+
uint8_t bits_are_set, int64_t length) {
|
|
2986
|
+
ArrowBitsSetTo(bitmap->buffer.data, bitmap->size_bits, length, bits_are_set);
|
|
2987
|
+
bitmap->size_bits += length;
|
|
2988
|
+
bitmap->buffer.size_bytes = _ArrowBytesForBits(bitmap->size_bits);
|
|
2989
|
+
}
|
|
2990
|
+
|
|
2991
|
+
static inline void ArrowBitmapAppendInt8Unsafe(struct ArrowBitmap* bitmap,
|
|
2992
|
+
const int8_t* values, int64_t n_values) {
|
|
2993
|
+
if (n_values == 0) {
|
|
2994
|
+
return;
|
|
2995
|
+
}
|
|
2996
|
+
|
|
2997
|
+
NANOARROW_DCHECK(bitmap->buffer.data != NULL);
|
|
2998
|
+
NANOARROW_DCHECK(values != NULL);
|
|
2999
|
+
|
|
3000
|
+
const int8_t* values_cursor = values;
|
|
3001
|
+
int64_t n_remaining = n_values;
|
|
3002
|
+
int64_t out_i_cursor = bitmap->size_bits;
|
|
3003
|
+
uint8_t* out_cursor = bitmap->buffer.data + bitmap->size_bits / 8;
|
|
3004
|
+
|
|
3005
|
+
// First byte
|
|
3006
|
+
if ((out_i_cursor % 8) != 0) {
|
|
3007
|
+
int64_t n_partial_bits = _ArrowRoundUpToMultipleOf8(out_i_cursor) - out_i_cursor;
|
|
3008
|
+
for (int i = 0; i < n_partial_bits; i++) {
|
|
3009
|
+
ArrowBitSetTo(bitmap->buffer.data, out_i_cursor++, values[i]);
|
|
3010
|
+
}
|
|
3011
|
+
|
|
3012
|
+
out_cursor++;
|
|
3013
|
+
values_cursor += n_partial_bits;
|
|
3014
|
+
n_remaining -= n_partial_bits;
|
|
3015
|
+
}
|
|
3016
|
+
|
|
3017
|
+
// Middle bytes
|
|
3018
|
+
int64_t n_full_bytes = n_remaining / 8;
|
|
3019
|
+
for (int64_t i = 0; i < n_full_bytes; i++) {
|
|
3020
|
+
_ArrowBitmapPackInt8(values_cursor, out_cursor);
|
|
3021
|
+
values_cursor += 8;
|
|
3022
|
+
out_cursor++;
|
|
3023
|
+
}
|
|
3024
|
+
|
|
3025
|
+
// Last byte
|
|
3026
|
+
out_i_cursor += n_full_bytes * 8;
|
|
3027
|
+
n_remaining -= n_full_bytes * 8;
|
|
3028
|
+
if (n_remaining > 0) {
|
|
3029
|
+
// Zero out the last byte
|
|
3030
|
+
*out_cursor = 0x00;
|
|
3031
|
+
for (int i = 0; i < n_remaining; i++) {
|
|
3032
|
+
ArrowBitSetTo(bitmap->buffer.data, out_i_cursor++, values_cursor[i]);
|
|
3033
|
+
}
|
|
3034
|
+
out_cursor++;
|
|
3035
|
+
}
|
|
3036
|
+
|
|
3037
|
+
bitmap->size_bits += n_values;
|
|
3038
|
+
bitmap->buffer.size_bytes = out_cursor - bitmap->buffer.data;
|
|
3039
|
+
}
|
|
3040
|
+
|
|
3041
|
+
static inline void ArrowBitmapAppendInt32Unsafe(struct ArrowBitmap* bitmap,
|
|
3042
|
+
const int32_t* values, int64_t n_values) {
|
|
3043
|
+
if (n_values == 0) {
|
|
3044
|
+
return;
|
|
3045
|
+
}
|
|
3046
|
+
|
|
3047
|
+
NANOARROW_DCHECK(bitmap->buffer.data != NULL);
|
|
3048
|
+
NANOARROW_DCHECK(values != NULL);
|
|
3049
|
+
|
|
3050
|
+
const int32_t* values_cursor = values;
|
|
3051
|
+
int64_t n_remaining = n_values;
|
|
3052
|
+
int64_t out_i_cursor = bitmap->size_bits;
|
|
3053
|
+
uint8_t* out_cursor = bitmap->buffer.data + bitmap->size_bits / 8;
|
|
3054
|
+
|
|
3055
|
+
// First byte
|
|
3056
|
+
if ((out_i_cursor % 8) != 0) {
|
|
3057
|
+
int64_t n_partial_bits = _ArrowRoundUpToMultipleOf8(out_i_cursor) - out_i_cursor;
|
|
3058
|
+
for (int i = 0; i < n_partial_bits; i++) {
|
|
3059
|
+
ArrowBitSetTo(bitmap->buffer.data, out_i_cursor++, (uint8_t)values[i]);
|
|
3060
|
+
}
|
|
3061
|
+
|
|
3062
|
+
out_cursor++;
|
|
3063
|
+
values_cursor += n_partial_bits;
|
|
3064
|
+
n_remaining -= n_partial_bits;
|
|
3065
|
+
}
|
|
3066
|
+
|
|
3067
|
+
// Middle bytes
|
|
3068
|
+
int64_t n_full_bytes = n_remaining / 8;
|
|
3069
|
+
for (int64_t i = 0; i < n_full_bytes; i++) {
|
|
3070
|
+
_ArrowBitmapPackInt32(values_cursor, out_cursor);
|
|
3071
|
+
values_cursor += 8;
|
|
3072
|
+
out_cursor++;
|
|
3073
|
+
}
|
|
3074
|
+
|
|
3075
|
+
// Last byte
|
|
3076
|
+
out_i_cursor += n_full_bytes * 8;
|
|
3077
|
+
n_remaining -= n_full_bytes * 8;
|
|
3078
|
+
if (n_remaining > 0) {
|
|
3079
|
+
// Zero out the last byte
|
|
3080
|
+
*out_cursor = 0x00;
|
|
3081
|
+
for (int i = 0; i < n_remaining; i++) {
|
|
3082
|
+
ArrowBitSetTo(bitmap->buffer.data, out_i_cursor++, (uint8_t)values_cursor[i]);
|
|
3083
|
+
}
|
|
3084
|
+
out_cursor++;
|
|
3085
|
+
}
|
|
3086
|
+
|
|
3087
|
+
bitmap->size_bits += n_values;
|
|
3088
|
+
bitmap->buffer.size_bytes = out_cursor - bitmap->buffer.data;
|
|
3089
|
+
}
|
|
3090
|
+
|
|
3091
|
+
static inline void ArrowBitmapReset(struct ArrowBitmap* bitmap) {
|
|
3092
|
+
ArrowBufferReset(&bitmap->buffer);
|
|
3093
|
+
bitmap->size_bits = 0;
|
|
3094
|
+
}
|
|
3095
|
+
|
|
3096
|
+
#ifdef __cplusplus
|
|
3097
|
+
}
|
|
3098
|
+
#endif
|
|
3099
|
+
|
|
3100
|
+
#endif
|
|
3101
|
+
// Licensed to the Apache Software Foundation (ASF) under one
|
|
3102
|
+
// or more contributor license agreements. See the NOTICE file
|
|
3103
|
+
// distributed with this work for additional information
|
|
3104
|
+
// regarding copyright ownership. The ASF licenses this file
|
|
3105
|
+
// to you under the Apache License, Version 2.0 (the
|
|
3106
|
+
// "License"); you may not use this file except in compliance
|
|
3107
|
+
// with the License. You may obtain a copy of the License at
|
|
3108
|
+
//
|
|
3109
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
3110
|
+
//
|
|
3111
|
+
// Unless required by applicable law or agreed to in writing,
|
|
3112
|
+
// software distributed under the License is distributed on an
|
|
3113
|
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
3114
|
+
// KIND, either express or implied. See the License for the
|
|
3115
|
+
// specific language governing permissions and limitations
|
|
3116
|
+
// under the License.
|
|
3117
|
+
|
|
3118
|
+
#ifndef NANOARROW_ARRAY_INLINE_H_INCLUDED
|
|
3119
|
+
#define NANOARROW_ARRAY_INLINE_H_INCLUDED
|
|
3120
|
+
|
|
3121
|
+
#include <errno.h>
|
|
3122
|
+
#include <float.h>
|
|
3123
|
+
#include <limits.h>
|
|
3124
|
+
#include <stdint.h>
|
|
3125
|
+
#include <string.h>
|
|
3126
|
+
|
|
3127
|
+
|
|
3128
|
+
|
|
3129
|
+
|
|
3130
|
+
#ifdef __cplusplus
|
|
3131
|
+
extern "C" {
|
|
3132
|
+
#endif
|
|
3133
|
+
|
|
3134
|
+
static inline struct ArrowBitmap* ArrowArrayValidityBitmap(struct ArrowArray* array) {
|
|
3135
|
+
struct ArrowArrayPrivateData* private_data =
|
|
3136
|
+
(struct ArrowArrayPrivateData*)array->private_data;
|
|
3137
|
+
return &private_data->bitmap;
|
|
3138
|
+
}
|
|
3139
|
+
|
|
3140
|
+
static inline struct ArrowBuffer* ArrowArrayBuffer(struct ArrowArray* array, int64_t i) {
|
|
3141
|
+
struct ArrowArrayPrivateData* private_data =
|
|
3142
|
+
(struct ArrowArrayPrivateData*)array->private_data;
|
|
3143
|
+
switch (i) {
|
|
3144
|
+
case 0:
|
|
3145
|
+
return &private_data->bitmap.buffer;
|
|
3146
|
+
case 1:
|
|
3147
|
+
return private_data->buffers;
|
|
3148
|
+
default:
|
|
3149
|
+
if (array->n_buffers > 3 && i == (array->n_buffers - 1)) {
|
|
3150
|
+
// The variadic buffer sizes buffer if for a BinaryView/String view array
|
|
3151
|
+
// is always stored in private_data->buffers[1]; however, from the numbered
|
|
3152
|
+
// buffers perspective this is the array->buffers[array->n_buffers - 1].
|
|
3153
|
+
return private_data->buffers + 1;
|
|
3154
|
+
} else if (array->n_buffers > 3) {
|
|
3155
|
+
// If there are one or more variadic buffers, they are stored in
|
|
3156
|
+
// private_data->variadic_buffers
|
|
3157
|
+
return private_data->variadic_buffers + (i - 2);
|
|
3158
|
+
} else {
|
|
3159
|
+
// Otherwise, we're just accessing buffer at index 2 (e.g., String/Binary
|
|
3160
|
+
// data buffer or variadic sizes buffer for the case where there are no
|
|
3161
|
+
// variadic buffers)
|
|
3162
|
+
NANOARROW_DCHECK(i == 2);
|
|
3163
|
+
return private_data->buffers + i - 1;
|
|
3164
|
+
}
|
|
3165
|
+
}
|
|
3166
|
+
}
|
|
3167
|
+
|
|
3168
|
+
// We don't currently support the case of unions where type_id != child_index;
|
|
3169
|
+
// however, these functions are used to keep track of where that assumption
|
|
3170
|
+
// is made.
|
|
3171
|
+
static inline int8_t _ArrowArrayUnionChildIndex(struct ArrowArray* array,
|
|
3172
|
+
int8_t type_id) {
|
|
3173
|
+
NANOARROW_UNUSED(array);
|
|
3174
|
+
return type_id;
|
|
3175
|
+
}
|
|
3176
|
+
|
|
3177
|
+
static inline int8_t _ArrowArrayUnionTypeId(struct ArrowArray* array,
|
|
3178
|
+
int8_t child_index) {
|
|
3179
|
+
NANOARROW_UNUSED(array);
|
|
3180
|
+
return child_index;
|
|
3181
|
+
}
|
|
3182
|
+
|
|
3183
|
+
static inline int32_t _ArrowParseUnionTypeIds(const char* type_ids, int8_t* out) {
|
|
3184
|
+
if (*type_ids == '\0') {
|
|
3185
|
+
return 0;
|
|
3186
|
+
}
|
|
3187
|
+
|
|
3188
|
+
int32_t i = 0;
|
|
3189
|
+
long type_id;
|
|
3190
|
+
char* end_ptr;
|
|
3191
|
+
do {
|
|
3192
|
+
type_id = strtol(type_ids, &end_ptr, 10);
|
|
3193
|
+
if (end_ptr == type_ids || type_id < 0 || type_id > 127) {
|
|
3194
|
+
return -1;
|
|
3195
|
+
}
|
|
3196
|
+
|
|
3197
|
+
if (out != NULL) {
|
|
3198
|
+
out[i] = (int8_t)type_id;
|
|
3199
|
+
}
|
|
3200
|
+
|
|
3201
|
+
i++;
|
|
3202
|
+
|
|
3203
|
+
type_ids = end_ptr;
|
|
3204
|
+
if (*type_ids == '\0') {
|
|
3205
|
+
return i;
|
|
3206
|
+
} else if (*type_ids != ',') {
|
|
3207
|
+
return -1;
|
|
3208
|
+
} else {
|
|
3209
|
+
type_ids++;
|
|
3210
|
+
}
|
|
3211
|
+
} while (1);
|
|
3212
|
+
|
|
3213
|
+
return -1;
|
|
3214
|
+
}
|
|
3215
|
+
|
|
3216
|
+
static inline int8_t _ArrowParsedUnionTypeIdsWillEqualChildIndices(const int8_t* type_ids,
|
|
3217
|
+
int64_t n_type_ids,
|
|
3218
|
+
int64_t n_children) {
|
|
3219
|
+
if (n_type_ids != n_children) {
|
|
3220
|
+
return 0;
|
|
3221
|
+
}
|
|
3222
|
+
|
|
3223
|
+
for (int8_t i = 0; i < n_type_ids; i++) {
|
|
3224
|
+
if (type_ids[i] != i) {
|
|
3225
|
+
return 0;
|
|
3226
|
+
}
|
|
3227
|
+
}
|
|
3228
|
+
|
|
3229
|
+
return 1;
|
|
3230
|
+
}
|
|
3231
|
+
|
|
3232
|
+
static inline int8_t _ArrowUnionTypeIdsWillEqualChildIndices(const char* type_id_str,
|
|
3233
|
+
int64_t n_children) {
|
|
3234
|
+
int8_t type_ids[128];
|
|
3235
|
+
int32_t n_type_ids = _ArrowParseUnionTypeIds(type_id_str, type_ids);
|
|
3236
|
+
return _ArrowParsedUnionTypeIdsWillEqualChildIndices(type_ids, n_type_ids, n_children);
|
|
3237
|
+
}
|
|
3238
|
+
|
|
3239
|
+
static inline ArrowErrorCode ArrowArrayStartAppending(struct ArrowArray* array) {
|
|
3240
|
+
struct ArrowArrayPrivateData* private_data =
|
|
3241
|
+
(struct ArrowArrayPrivateData*)array->private_data;
|
|
3242
|
+
|
|
3243
|
+
switch (private_data->storage_type) {
|
|
3244
|
+
case NANOARROW_TYPE_UNINITIALIZED:
|
|
3245
|
+
return EINVAL;
|
|
3246
|
+
case NANOARROW_TYPE_SPARSE_UNION:
|
|
3247
|
+
case NANOARROW_TYPE_DENSE_UNION:
|
|
3248
|
+
// Note that this value could be -1 if the type_ids string was invalid
|
|
3249
|
+
if (private_data->union_type_id_is_child_index != 1) {
|
|
3250
|
+
return EINVAL;
|
|
3251
|
+
} else {
|
|
3252
|
+
break;
|
|
3253
|
+
}
|
|
3254
|
+
default:
|
|
3255
|
+
break;
|
|
3256
|
+
}
|
|
3257
|
+
if (private_data->storage_type == NANOARROW_TYPE_UNINITIALIZED) {
|
|
3258
|
+
return EINVAL;
|
|
3259
|
+
}
|
|
3260
|
+
|
|
3261
|
+
// Initialize any data offset buffer with a single zero
|
|
3262
|
+
for (int i = 0; i < NANOARROW_MAX_FIXED_BUFFERS; i++) {
|
|
3263
|
+
if (private_data->layout.buffer_type[i] == NANOARROW_BUFFER_TYPE_DATA_OFFSET &&
|
|
3264
|
+
private_data->layout.element_size_bits[i] == 64) {
|
|
3265
|
+
NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt64(ArrowArrayBuffer(array, i), 0));
|
|
3266
|
+
} else if (private_data->layout.buffer_type[i] == NANOARROW_BUFFER_TYPE_DATA_OFFSET &&
|
|
3267
|
+
private_data->layout.element_size_bits[i] == 32) {
|
|
3268
|
+
NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(ArrowArrayBuffer(array, i), 0));
|
|
3269
|
+
}
|
|
3270
|
+
}
|
|
3271
|
+
|
|
3272
|
+
// Start building any child arrays or dictionaries
|
|
3273
|
+
for (int64_t i = 0; i < array->n_children; i++) {
|
|
3274
|
+
NANOARROW_RETURN_NOT_OK(ArrowArrayStartAppending(array->children[i]));
|
|
3275
|
+
}
|
|
3276
|
+
|
|
3277
|
+
if (array->dictionary != NULL) {
|
|
3278
|
+
NANOARROW_RETURN_NOT_OK(ArrowArrayStartAppending(array->dictionary));
|
|
3279
|
+
}
|
|
3280
|
+
|
|
3281
|
+
return NANOARROW_OK;
|
|
3282
|
+
}
|
|
3283
|
+
|
|
3284
|
+
static inline ArrowErrorCode ArrowArrayShrinkToFit(struct ArrowArray* array) {
|
|
3285
|
+
for (int64_t i = 0; i < NANOARROW_MAX_FIXED_BUFFERS; i++) {
|
|
3286
|
+
struct ArrowBuffer* buffer = ArrowArrayBuffer(array, i);
|
|
3287
|
+
NANOARROW_RETURN_NOT_OK(ArrowBufferResize(buffer, buffer->size_bytes, 1));
|
|
3288
|
+
}
|
|
3289
|
+
|
|
3290
|
+
for (int64_t i = 0; i < array->n_children; i++) {
|
|
3291
|
+
NANOARROW_RETURN_NOT_OK(ArrowArrayShrinkToFit(array->children[i]));
|
|
3292
|
+
}
|
|
3293
|
+
|
|
3294
|
+
if (array->dictionary != NULL) {
|
|
3295
|
+
NANOARROW_RETURN_NOT_OK(ArrowArrayShrinkToFit(array->dictionary));
|
|
3296
|
+
}
|
|
3297
|
+
|
|
3298
|
+
return NANOARROW_OK;
|
|
3299
|
+
}
|
|
3300
|
+
|
|
3301
|
+
static inline ArrowErrorCode _ArrowArrayAppendBits(struct ArrowArray* array,
|
|
3302
|
+
int64_t buffer_i, uint8_t value,
|
|
3303
|
+
int64_t n) {
|
|
3304
|
+
struct ArrowArrayPrivateData* private_data =
|
|
3305
|
+
(struct ArrowArrayPrivateData*)array->private_data;
|
|
3306
|
+
struct ArrowBuffer* buffer = ArrowArrayBuffer(array, buffer_i);
|
|
3307
|
+
int64_t bytes_required =
|
|
3308
|
+
_ArrowRoundUpToMultipleOf8(private_data->layout.element_size_bits[buffer_i] *
|
|
3309
|
+
(array->length + 1)) /
|
|
3310
|
+
8;
|
|
3311
|
+
if (bytes_required > buffer->size_bytes) {
|
|
3312
|
+
NANOARROW_RETURN_NOT_OK(
|
|
3313
|
+
ArrowBufferAppendFill(buffer, 0, bytes_required - buffer->size_bytes));
|
|
3314
|
+
}
|
|
3315
|
+
|
|
3316
|
+
ArrowBitsSetTo(buffer->data, array->length, n, value);
|
|
3317
|
+
return NANOARROW_OK;
|
|
3318
|
+
}
|
|
3319
|
+
|
|
3320
|
+
static inline ArrowErrorCode _ArrowArrayAppendEmptyInternal(struct ArrowArray* array,
|
|
3321
|
+
int64_t n, uint8_t is_valid) {
|
|
3322
|
+
struct ArrowArrayPrivateData* private_data =
|
|
3323
|
+
(struct ArrowArrayPrivateData*)array->private_data;
|
|
3324
|
+
|
|
3325
|
+
if (n == 0) {
|
|
3326
|
+
return NANOARROW_OK;
|
|
3327
|
+
}
|
|
3328
|
+
|
|
3329
|
+
// Some type-specific handling
|
|
3330
|
+
switch (private_data->storage_type) {
|
|
3331
|
+
case NANOARROW_TYPE_NA:
|
|
3332
|
+
// (An empty value for a null array *is* a null)
|
|
3333
|
+
array->null_count += n;
|
|
3334
|
+
array->length += n;
|
|
3335
|
+
return NANOARROW_OK;
|
|
3336
|
+
|
|
3337
|
+
case NANOARROW_TYPE_DENSE_UNION: {
|
|
3338
|
+
// Add one null to the first child and append n references to that child
|
|
3339
|
+
int8_t type_id = _ArrowArrayUnionTypeId(array, 0);
|
|
3340
|
+
NANOARROW_RETURN_NOT_OK(
|
|
3341
|
+
_ArrowArrayAppendEmptyInternal(array->children[0], 1, is_valid));
|
|
3342
|
+
NANOARROW_RETURN_NOT_OK(
|
|
3343
|
+
ArrowBufferAppendFill(ArrowArrayBuffer(array, 0), type_id, n));
|
|
3344
|
+
for (int64_t i = 0; i < n; i++) {
|
|
3345
|
+
NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(
|
|
3346
|
+
ArrowArrayBuffer(array, 1), (int32_t)array->children[0]->length - 1));
|
|
3347
|
+
}
|
|
3348
|
+
// For the purposes of array->null_count, union elements are never considered "null"
|
|
3349
|
+
// even if some children contain nulls.
|
|
3350
|
+
array->length += n;
|
|
3351
|
+
return NANOARROW_OK;
|
|
3352
|
+
}
|
|
3353
|
+
|
|
3354
|
+
case NANOARROW_TYPE_SPARSE_UNION: {
|
|
3355
|
+
// Add n nulls to the first child and append n references to that child
|
|
3356
|
+
int8_t type_id = _ArrowArrayUnionTypeId(array, 0);
|
|
3357
|
+
NANOARROW_RETURN_NOT_OK(
|
|
3358
|
+
_ArrowArrayAppendEmptyInternal(array->children[0], n, is_valid));
|
|
3359
|
+
for (int64_t i = 1; i < array->n_children; i++) {
|
|
3360
|
+
NANOARROW_RETURN_NOT_OK(ArrowArrayAppendEmpty(array->children[i], n));
|
|
3361
|
+
}
|
|
3362
|
+
|
|
3363
|
+
NANOARROW_RETURN_NOT_OK(
|
|
3364
|
+
ArrowBufferAppendFill(ArrowArrayBuffer(array, 0), type_id, n));
|
|
3365
|
+
// For the purposes of array->null_count, union elements are never considered "null"
|
|
3366
|
+
// even if some children contain nulls.
|
|
3367
|
+
array->length += n;
|
|
3368
|
+
return NANOARROW_OK;
|
|
3369
|
+
}
|
|
3370
|
+
|
|
3371
|
+
case NANOARROW_TYPE_FIXED_SIZE_LIST:
|
|
3372
|
+
NANOARROW_RETURN_NOT_OK(ArrowArrayAppendEmpty(
|
|
3373
|
+
array->children[0], n * private_data->layout.child_size_elements));
|
|
3374
|
+
break;
|
|
3375
|
+
case NANOARROW_TYPE_STRUCT:
|
|
3376
|
+
for (int64_t i = 0; i < array->n_children; i++) {
|
|
3377
|
+
NANOARROW_RETURN_NOT_OK(ArrowArrayAppendEmpty(array->children[i], n));
|
|
3378
|
+
}
|
|
3379
|
+
break;
|
|
3380
|
+
|
|
3381
|
+
default:
|
|
3382
|
+
break;
|
|
3383
|
+
}
|
|
3384
|
+
|
|
3385
|
+
// Append n is_valid bits to the validity bitmap. If we haven't allocated a bitmap yet
|
|
3386
|
+
// and we need to append nulls, do it now.
|
|
3387
|
+
if (!is_valid && private_data->bitmap.buffer.data == NULL) {
|
|
3388
|
+
NANOARROW_RETURN_NOT_OK(ArrowBitmapReserve(&private_data->bitmap, array->length + n));
|
|
3389
|
+
ArrowBitmapAppendUnsafe(&private_data->bitmap, 1, array->length);
|
|
3390
|
+
ArrowBitmapAppendUnsafe(&private_data->bitmap, is_valid, n);
|
|
3391
|
+
} else if (private_data->bitmap.buffer.data != NULL) {
|
|
3392
|
+
NANOARROW_RETURN_NOT_OK(ArrowBitmapReserve(&private_data->bitmap, n));
|
|
3393
|
+
ArrowBitmapAppendUnsafe(&private_data->bitmap, is_valid, n);
|
|
3394
|
+
}
|
|
3395
|
+
|
|
3396
|
+
// Add appropriate buffer fill
|
|
3397
|
+
for (int i = 0; i < NANOARROW_MAX_FIXED_BUFFERS; i++) {
|
|
3398
|
+
struct ArrowBuffer* buffer = ArrowArrayBuffer(array, i);
|
|
3399
|
+
int64_t size_bytes = private_data->layout.element_size_bits[i] / 8;
|
|
3400
|
+
|
|
3401
|
+
switch (private_data->layout.buffer_type[i]) {
|
|
3402
|
+
case NANOARROW_BUFFER_TYPE_NONE:
|
|
3403
|
+
case NANOARROW_BUFFER_TYPE_VARIADIC_DATA:
|
|
3404
|
+
case NANOARROW_BUFFER_TYPE_VARIADIC_SIZE:
|
|
3405
|
+
case NANOARROW_BUFFER_TYPE_VALIDITY:
|
|
3406
|
+
// These buffer types don't require initialization for empty appends:
|
|
3407
|
+
// - NONE: No buffer exists
|
|
3408
|
+
// - VARIADIC_*: Handled by child arrays
|
|
3409
|
+
// - VALIDITY: Already handled in previous bitmap logic
|
|
3410
|
+
break;
|
|
3411
|
+
|
|
3412
|
+
case NANOARROW_BUFFER_TYPE_SIZE:
|
|
3413
|
+
// Size buffers (e.g., string/array lengths) should be zero-initialized:
|
|
3414
|
+
// This ensures empty elements have logical zero-length
|
|
3415
|
+
NANOARROW_RETURN_NOT_OK(ArrowBufferAppendFill(buffer, 0, size_bytes * n));
|
|
3416
|
+
break;
|
|
3417
|
+
|
|
3418
|
+
case NANOARROW_BUFFER_TYPE_DATA_OFFSET:
|
|
3419
|
+
// Offset buffers require special handling to maintain continuity.
|
|
3420
|
+
// 1. Reserve space for new offset entries
|
|
3421
|
+
NANOARROW_RETURN_NOT_OK(ArrowBufferReserve(buffer, size_bytes * n));
|
|
3422
|
+
|
|
3423
|
+
// 2. Duplicate last offset value for each new (empty) element
|
|
3424
|
+
for (int64_t j = 0; j < n; j++) {
|
|
3425
|
+
ArrowBufferAppendUnsafe(buffer, buffer->data + size_bytes * (array->length + j),
|
|
3426
|
+
size_bytes);
|
|
3427
|
+
}
|
|
3428
|
+
|
|
3429
|
+
// 3. Skip next buffer (DATA) since it's paired with offsets
|
|
3430
|
+
// Rationale: Offset buffers are always followed by data buffers
|
|
3431
|
+
// that don't require separate initialization here
|
|
3432
|
+
i++;
|
|
3433
|
+
break;
|
|
3434
|
+
|
|
3435
|
+
case NANOARROW_BUFFER_TYPE_DATA:
|
|
3436
|
+
// Fixed-width data buffers require zero-initialization:
|
|
3437
|
+
if (private_data->layout.element_size_bits[i] % 8 == 0) {
|
|
3438
|
+
// Byte-aligned: use efficient memset-style fill
|
|
3439
|
+
NANOARROW_RETURN_NOT_OK(ArrowBufferAppendFill(buffer, 0, size_bytes * n));
|
|
3440
|
+
} else {
|
|
3441
|
+
// Bit-packed: use special bitwise initialization
|
|
3442
|
+
NANOARROW_RETURN_NOT_OK(_ArrowArrayAppendBits(array, i, 0, n));
|
|
3443
|
+
}
|
|
3444
|
+
break;
|
|
3445
|
+
|
|
3446
|
+
case NANOARROW_BUFFER_TYPE_VIEW_OFFSET:
|
|
3447
|
+
// View offset buffers (for string/binary view types) require zero-initialization.
|
|
3448
|
+
NANOARROW_RETURN_NOT_OK(ArrowBufferReserve(buffer, size_bytes * n));
|
|
3449
|
+
NANOARROW_RETURN_NOT_OK(ArrowBufferAppendFill(buffer, 0, size_bytes * n));
|
|
3450
|
+
break;
|
|
3451
|
+
|
|
3452
|
+
case NANOARROW_BUFFER_TYPE_TYPE_ID:
|
|
3453
|
+
case NANOARROW_BUFFER_TYPE_UNION_OFFSET:
|
|
3454
|
+
// These buffer types should have been handled by the outer type switch and
|
|
3455
|
+
// are not expected here, indicating an internal logic error.
|
|
3456
|
+
return EINVAL;
|
|
3457
|
+
}
|
|
3458
|
+
}
|
|
3459
|
+
|
|
3460
|
+
array->length += n;
|
|
3461
|
+
array->null_count += n * !is_valid;
|
|
3462
|
+
return NANOARROW_OK;
|
|
3463
|
+
}
|
|
3464
|
+
|
|
3465
|
+
static inline ArrowErrorCode ArrowArrayAppendNull(struct ArrowArray* array, int64_t n) {
|
|
3466
|
+
return _ArrowArrayAppendEmptyInternal(array, n, 0);
|
|
3467
|
+
}
|
|
3468
|
+
|
|
3469
|
+
static inline ArrowErrorCode ArrowArrayAppendEmpty(struct ArrowArray* array, int64_t n) {
|
|
3470
|
+
return _ArrowArrayAppendEmptyInternal(array, n, 1);
|
|
3471
|
+
}
|
|
3472
|
+
|
|
3473
|
+
static inline ArrowErrorCode ArrowArrayAppendInt(struct ArrowArray* array,
|
|
3474
|
+
int64_t value) {
|
|
3475
|
+
struct ArrowArrayPrivateData* private_data =
|
|
3476
|
+
(struct ArrowArrayPrivateData*)array->private_data;
|
|
3477
|
+
|
|
3478
|
+
struct ArrowBuffer* data_buffer = ArrowArrayBuffer(array, 1);
|
|
3479
|
+
|
|
3480
|
+
switch (private_data->storage_type) {
|
|
3481
|
+
case NANOARROW_TYPE_INT64:
|
|
3482
|
+
NANOARROW_RETURN_NOT_OK(ArrowBufferAppend(data_buffer, &value, sizeof(int64_t)));
|
|
3483
|
+
break;
|
|
3484
|
+
case NANOARROW_TYPE_INT32:
|
|
3485
|
+
_NANOARROW_CHECK_RANGE(value, INT32_MIN, INT32_MAX);
|
|
3486
|
+
NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(data_buffer, (int32_t)value));
|
|
3487
|
+
break;
|
|
3488
|
+
case NANOARROW_TYPE_INT16:
|
|
3489
|
+
_NANOARROW_CHECK_RANGE(value, INT16_MIN, INT16_MAX);
|
|
3490
|
+
NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt16(data_buffer, (int16_t)value));
|
|
3491
|
+
break;
|
|
3492
|
+
case NANOARROW_TYPE_INT8:
|
|
3493
|
+
_NANOARROW_CHECK_RANGE(value, INT8_MIN, INT8_MAX);
|
|
3494
|
+
NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt8(data_buffer, (int8_t)value));
|
|
3495
|
+
break;
|
|
3496
|
+
case NANOARROW_TYPE_UINT64:
|
|
3497
|
+
case NANOARROW_TYPE_UINT32:
|
|
3498
|
+
case NANOARROW_TYPE_UINT16:
|
|
3499
|
+
case NANOARROW_TYPE_UINT8:
|
|
3500
|
+
_NANOARROW_CHECK_RANGE(value, 0, INT64_MAX);
|
|
3501
|
+
return ArrowArrayAppendUInt(array, value);
|
|
3502
|
+
case NANOARROW_TYPE_DOUBLE:
|
|
3503
|
+
NANOARROW_RETURN_NOT_OK(ArrowBufferAppendDouble(data_buffer, (double)value));
|
|
3504
|
+
break;
|
|
3505
|
+
case NANOARROW_TYPE_FLOAT:
|
|
3506
|
+
NANOARROW_RETURN_NOT_OK(ArrowBufferAppendFloat(data_buffer, (float)value));
|
|
3507
|
+
break;
|
|
3508
|
+
case NANOARROW_TYPE_HALF_FLOAT:
|
|
3509
|
+
NANOARROW_RETURN_NOT_OK(
|
|
3510
|
+
ArrowBufferAppendUInt16(data_buffer, ArrowFloatToHalfFloat((float)value)));
|
|
3511
|
+
break;
|
|
3512
|
+
case NANOARROW_TYPE_BOOL:
|
|
3513
|
+
NANOARROW_RETURN_NOT_OK(_ArrowArrayAppendBits(array, 1, value != 0, 1));
|
|
3514
|
+
break;
|
|
3515
|
+
default:
|
|
3516
|
+
return EINVAL;
|
|
3517
|
+
}
|
|
3518
|
+
|
|
3519
|
+
if (private_data->bitmap.buffer.data != NULL) {
|
|
3520
|
+
NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1));
|
|
3521
|
+
}
|
|
3522
|
+
|
|
3523
|
+
array->length++;
|
|
3524
|
+
return NANOARROW_OK;
|
|
3525
|
+
}
|
|
3526
|
+
|
|
3527
|
+
static inline ArrowErrorCode ArrowArrayAppendUInt(struct ArrowArray* array,
|
|
3528
|
+
uint64_t value) {
|
|
3529
|
+
struct ArrowArrayPrivateData* private_data =
|
|
3530
|
+
(struct ArrowArrayPrivateData*)array->private_data;
|
|
3531
|
+
|
|
3532
|
+
struct ArrowBuffer* data_buffer = ArrowArrayBuffer(array, 1);
|
|
3533
|
+
|
|
3534
|
+
switch (private_data->storage_type) {
|
|
3535
|
+
case NANOARROW_TYPE_UINT64:
|
|
3536
|
+
NANOARROW_RETURN_NOT_OK(ArrowBufferAppend(data_buffer, &value, sizeof(uint64_t)));
|
|
3537
|
+
break;
|
|
3538
|
+
case NANOARROW_TYPE_UINT32:
|
|
3539
|
+
_NANOARROW_CHECK_UPPER_LIMIT(value, UINT32_MAX);
|
|
3540
|
+
NANOARROW_RETURN_NOT_OK(ArrowBufferAppendUInt32(data_buffer, (uint32_t)value));
|
|
3541
|
+
break;
|
|
3542
|
+
case NANOARROW_TYPE_UINT16:
|
|
3543
|
+
_NANOARROW_CHECK_UPPER_LIMIT(value, UINT16_MAX);
|
|
3544
|
+
NANOARROW_RETURN_NOT_OK(ArrowBufferAppendUInt16(data_buffer, (uint16_t)value));
|
|
3545
|
+
break;
|
|
3546
|
+
case NANOARROW_TYPE_UINT8:
|
|
3547
|
+
_NANOARROW_CHECK_UPPER_LIMIT(value, UINT8_MAX);
|
|
3548
|
+
NANOARROW_RETURN_NOT_OK(ArrowBufferAppendUInt8(data_buffer, (uint8_t)value));
|
|
3549
|
+
break;
|
|
3550
|
+
case NANOARROW_TYPE_INT64:
|
|
3551
|
+
case NANOARROW_TYPE_INT32:
|
|
3552
|
+
case NANOARROW_TYPE_INT16:
|
|
3553
|
+
case NANOARROW_TYPE_INT8:
|
|
3554
|
+
_NANOARROW_CHECK_UPPER_LIMIT(value, INT64_MAX);
|
|
3555
|
+
return ArrowArrayAppendInt(array, value);
|
|
3556
|
+
case NANOARROW_TYPE_DOUBLE:
|
|
3557
|
+
NANOARROW_RETURN_NOT_OK(ArrowBufferAppendDouble(data_buffer, (double)value));
|
|
3558
|
+
break;
|
|
3559
|
+
case NANOARROW_TYPE_FLOAT:
|
|
3560
|
+
NANOARROW_RETURN_NOT_OK(ArrowBufferAppendFloat(data_buffer, (float)value));
|
|
3561
|
+
break;
|
|
3562
|
+
case NANOARROW_TYPE_HALF_FLOAT:
|
|
3563
|
+
NANOARROW_RETURN_NOT_OK(
|
|
3564
|
+
ArrowBufferAppendUInt16(data_buffer, ArrowFloatToHalfFloat((float)value)));
|
|
3565
|
+
break;
|
|
3566
|
+
case NANOARROW_TYPE_BOOL:
|
|
3567
|
+
NANOARROW_RETURN_NOT_OK(_ArrowArrayAppendBits(array, 1, value != 0, 1));
|
|
3568
|
+
break;
|
|
3569
|
+
default:
|
|
3570
|
+
return EINVAL;
|
|
3571
|
+
}
|
|
3572
|
+
|
|
3573
|
+
if (private_data->bitmap.buffer.data != NULL) {
|
|
3574
|
+
NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1));
|
|
3575
|
+
}
|
|
3576
|
+
|
|
3577
|
+
array->length++;
|
|
3578
|
+
return NANOARROW_OK;
|
|
3579
|
+
}
|
|
3580
|
+
|
|
3581
|
+
static inline ArrowErrorCode ArrowArrayAppendDouble(struct ArrowArray* array,
|
|
3582
|
+
double value) {
|
|
3583
|
+
struct ArrowArrayPrivateData* private_data =
|
|
3584
|
+
(struct ArrowArrayPrivateData*)array->private_data;
|
|
3585
|
+
|
|
3586
|
+
struct ArrowBuffer* data_buffer = ArrowArrayBuffer(array, 1);
|
|
3587
|
+
|
|
3588
|
+
switch (private_data->storage_type) {
|
|
3589
|
+
case NANOARROW_TYPE_DOUBLE:
|
|
3590
|
+
NANOARROW_RETURN_NOT_OK(ArrowBufferAppend(data_buffer, &value, sizeof(double)));
|
|
3591
|
+
break;
|
|
3592
|
+
case NANOARROW_TYPE_FLOAT:
|
|
3593
|
+
NANOARROW_RETURN_NOT_OK(ArrowBufferAppendFloat(data_buffer, (float)value));
|
|
3594
|
+
break;
|
|
3595
|
+
case NANOARROW_TYPE_HALF_FLOAT:
|
|
3596
|
+
NANOARROW_RETURN_NOT_OK(
|
|
3597
|
+
ArrowBufferAppendUInt16(data_buffer, ArrowFloatToHalfFloat((float)value)));
|
|
3598
|
+
break;
|
|
3599
|
+
default:
|
|
3600
|
+
return EINVAL;
|
|
3601
|
+
}
|
|
3602
|
+
|
|
3603
|
+
if (private_data->bitmap.buffer.data != NULL) {
|
|
3604
|
+
NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1));
|
|
3605
|
+
}
|
|
3606
|
+
|
|
3607
|
+
array->length++;
|
|
3608
|
+
return NANOARROW_OK;
|
|
3609
|
+
}
|
|
3610
|
+
|
|
3611
|
+
// Binary views only have two fixed buffers, but be aware that they must also
|
|
3612
|
+
// always have more 1 buffer to store variadic buffer sizes (even if there are none)
|
|
3613
|
+
#define NANOARROW_BINARY_VIEW_FIXED_BUFFERS 2
|
|
3614
|
+
#define NANOARROW_BINARY_VIEW_INLINE_SIZE 12
|
|
3615
|
+
#define NANOARROW_BINARY_VIEW_PREFIX_SIZE 4
|
|
3616
|
+
#define NANOARROW_BINARY_VIEW_BLOCK_SIZE (32 << 10) // 32KB
|
|
3617
|
+
|
|
3618
|
+
// The Arrow C++ implementation uses anonymous structs as members
|
|
3619
|
+
// of the ArrowBinaryView. For Cython support in this library, we define
|
|
3620
|
+
// those structs outside of the ArrowBinaryView
|
|
3621
|
+
struct ArrowBinaryViewInlined {
|
|
3622
|
+
int32_t size;
|
|
3623
|
+
uint8_t data[NANOARROW_BINARY_VIEW_INLINE_SIZE];
|
|
3624
|
+
};
|
|
3625
|
+
|
|
3626
|
+
struct ArrowBinaryViewRef {
|
|
3627
|
+
int32_t size;
|
|
3628
|
+
uint8_t prefix[NANOARROW_BINARY_VIEW_PREFIX_SIZE];
|
|
3629
|
+
int32_t buffer_index;
|
|
3630
|
+
int32_t offset;
|
|
3631
|
+
};
|
|
3632
|
+
|
|
3633
|
+
union ArrowBinaryView {
|
|
3634
|
+
struct ArrowBinaryViewInlined inlined;
|
|
3635
|
+
struct ArrowBinaryViewRef ref;
|
|
3636
|
+
int64_t alignment_dummy;
|
|
3637
|
+
};
|
|
3638
|
+
|
|
3639
|
+
static inline int32_t ArrowArrayVariadicBufferCount(struct ArrowArray* array) {
|
|
3640
|
+
struct ArrowArrayPrivateData* private_data =
|
|
3641
|
+
(struct ArrowArrayPrivateData*)array->private_data;
|
|
3642
|
+
|
|
3643
|
+
return private_data->n_variadic_buffers;
|
|
3644
|
+
}
|
|
3645
|
+
|
|
3646
|
+
static inline ArrowErrorCode ArrowArrayAddVariadicBuffers(struct ArrowArray* array,
|
|
3647
|
+
int32_t n_buffers) {
|
|
3648
|
+
const int32_t n_current_bufs = ArrowArrayVariadicBufferCount(array);
|
|
3649
|
+
const int32_t nvariadic_bufs_needed = n_current_bufs + n_buffers;
|
|
3650
|
+
|
|
3651
|
+
struct ArrowArrayPrivateData* private_data =
|
|
3652
|
+
(struct ArrowArrayPrivateData*)array->private_data;
|
|
3653
|
+
|
|
3654
|
+
private_data->variadic_buffers = (struct ArrowBuffer*)ArrowRealloc(
|
|
3655
|
+
private_data->variadic_buffers, sizeof(struct ArrowBuffer) * nvariadic_bufs_needed);
|
|
3656
|
+
if (private_data->variadic_buffers == NULL) {
|
|
3657
|
+
return ENOMEM;
|
|
3658
|
+
}
|
|
3659
|
+
|
|
3660
|
+
private_data->n_variadic_buffers = nvariadic_bufs_needed;
|
|
3661
|
+
array->n_buffers = NANOARROW_BINARY_VIEW_FIXED_BUFFERS + 1 + nvariadic_bufs_needed;
|
|
3662
|
+
|
|
3663
|
+
private_data->buffer_data = (const void**)ArrowRealloc(
|
|
3664
|
+
private_data->buffer_data, array->n_buffers * sizeof(void*));
|
|
3665
|
+
|
|
3666
|
+
for (int32_t i = n_current_bufs; i < nvariadic_bufs_needed; i++) {
|
|
3667
|
+
ArrowBufferInit(&private_data->variadic_buffers[i]);
|
|
3668
|
+
private_data->buffer_data[NANOARROW_BINARY_VIEW_FIXED_BUFFERS + i] = NULL;
|
|
3669
|
+
}
|
|
3670
|
+
|
|
3671
|
+
// Zero out memory for the final buffer (variadic sizes buffer we haven't built yet)
|
|
3672
|
+
private_data->buffer_data[NANOARROW_BINARY_VIEW_FIXED_BUFFERS + nvariadic_bufs_needed] =
|
|
3673
|
+
NULL;
|
|
3674
|
+
|
|
3675
|
+
// Ensure array->buffers points to a valid value
|
|
3676
|
+
array->buffers = private_data->buffer_data;
|
|
3677
|
+
return NANOARROW_OK;
|
|
3678
|
+
}
|
|
3679
|
+
|
|
3680
|
+
static inline ArrowErrorCode ArrowArrayAppendBytes(struct ArrowArray* array,
|
|
3681
|
+
struct ArrowBufferView value) {
|
|
3682
|
+
struct ArrowArrayPrivateData* private_data =
|
|
3683
|
+
(struct ArrowArrayPrivateData*)array->private_data;
|
|
3684
|
+
|
|
3685
|
+
if (private_data->storage_type == NANOARROW_TYPE_STRING_VIEW ||
|
|
3686
|
+
private_data->storage_type == NANOARROW_TYPE_BINARY_VIEW) {
|
|
3687
|
+
struct ArrowBuffer* data_buffer = ArrowArrayBuffer(array, 1);
|
|
3688
|
+
union ArrowBinaryView bvt;
|
|
3689
|
+
bvt.inlined.size = (int32_t)value.size_bytes;
|
|
3690
|
+
|
|
3691
|
+
if (value.size_bytes <= NANOARROW_BINARY_VIEW_INLINE_SIZE) {
|
|
3692
|
+
memcpy(bvt.inlined.data, value.data.as_char, value.size_bytes);
|
|
3693
|
+
memset(bvt.inlined.data + bvt.inlined.size, 0,
|
|
3694
|
+
NANOARROW_BINARY_VIEW_INLINE_SIZE - bvt.inlined.size);
|
|
3695
|
+
} else {
|
|
3696
|
+
int32_t current_n_vbufs = ArrowArrayVariadicBufferCount(array);
|
|
3697
|
+
if (current_n_vbufs == 0 ||
|
|
3698
|
+
private_data->variadic_buffers[current_n_vbufs - 1].size_bytes +
|
|
3699
|
+
value.size_bytes >
|
|
3700
|
+
NANOARROW_BINARY_VIEW_BLOCK_SIZE) {
|
|
3701
|
+
const int32_t additional_bufs_needed = 1;
|
|
3702
|
+
NANOARROW_RETURN_NOT_OK(
|
|
3703
|
+
ArrowArrayAddVariadicBuffers(array, additional_bufs_needed));
|
|
3704
|
+
current_n_vbufs += additional_bufs_needed;
|
|
3705
|
+
}
|
|
3706
|
+
|
|
3707
|
+
const int32_t buf_index = current_n_vbufs - 1;
|
|
3708
|
+
struct ArrowBuffer* variadic_buf = &private_data->variadic_buffers[buf_index];
|
|
3709
|
+
memcpy(bvt.ref.prefix, value.data.as_char, NANOARROW_BINARY_VIEW_PREFIX_SIZE);
|
|
3710
|
+
bvt.ref.buffer_index = (int32_t)buf_index;
|
|
3711
|
+
bvt.ref.offset = (int32_t)variadic_buf->size_bytes;
|
|
3712
|
+
NANOARROW_RETURN_NOT_OK(
|
|
3713
|
+
ArrowBufferAppend(variadic_buf, value.data.as_char, value.size_bytes));
|
|
3714
|
+
}
|
|
3715
|
+
NANOARROW_RETURN_NOT_OK(ArrowBufferAppend(data_buffer, &bvt, sizeof(bvt)));
|
|
3716
|
+
} else {
|
|
3717
|
+
struct ArrowBuffer* offset_buffer = ArrowArrayBuffer(array, 1);
|
|
3718
|
+
struct ArrowBuffer* data_buffer = ArrowArrayBuffer(
|
|
3719
|
+
array, 1 + (private_data->storage_type != NANOARROW_TYPE_FIXED_SIZE_BINARY));
|
|
3720
|
+
int32_t offset;
|
|
3721
|
+
int64_t large_offset;
|
|
3722
|
+
int64_t fixed_size_bytes = private_data->layout.element_size_bits[1] / 8;
|
|
3723
|
+
|
|
3724
|
+
switch (private_data->storage_type) {
|
|
3725
|
+
case NANOARROW_TYPE_STRING:
|
|
3726
|
+
case NANOARROW_TYPE_BINARY:
|
|
3727
|
+
offset = ((int32_t*)offset_buffer->data)[array->length];
|
|
3728
|
+
if ((((int64_t)offset) + value.size_bytes) > INT32_MAX) {
|
|
3729
|
+
return EOVERFLOW;
|
|
3730
|
+
}
|
|
3731
|
+
|
|
3732
|
+
offset += (int32_t)value.size_bytes;
|
|
3733
|
+
NANOARROW_RETURN_NOT_OK(
|
|
3734
|
+
ArrowBufferAppend(offset_buffer, &offset, sizeof(int32_t)));
|
|
3735
|
+
NANOARROW_RETURN_NOT_OK(
|
|
3736
|
+
ArrowBufferAppend(data_buffer, value.data.data, value.size_bytes));
|
|
3737
|
+
break;
|
|
3738
|
+
|
|
3739
|
+
case NANOARROW_TYPE_LARGE_STRING:
|
|
3740
|
+
case NANOARROW_TYPE_LARGE_BINARY:
|
|
3741
|
+
large_offset = ((int64_t*)offset_buffer->data)[array->length];
|
|
3742
|
+
large_offset += value.size_bytes;
|
|
3743
|
+
NANOARROW_RETURN_NOT_OK(
|
|
3744
|
+
ArrowBufferAppend(offset_buffer, &large_offset, sizeof(int64_t)));
|
|
3745
|
+
NANOARROW_RETURN_NOT_OK(
|
|
3746
|
+
ArrowBufferAppend(data_buffer, value.data.data, value.size_bytes));
|
|
3747
|
+
break;
|
|
3748
|
+
|
|
3749
|
+
case NANOARROW_TYPE_FIXED_SIZE_BINARY:
|
|
3750
|
+
if (value.size_bytes != fixed_size_bytes) {
|
|
3751
|
+
return EINVAL;
|
|
3752
|
+
}
|
|
3753
|
+
|
|
3754
|
+
NANOARROW_RETURN_NOT_OK(
|
|
3755
|
+
ArrowBufferAppend(data_buffer, value.data.data, value.size_bytes));
|
|
3756
|
+
break;
|
|
3757
|
+
default:
|
|
3758
|
+
return EINVAL;
|
|
3759
|
+
}
|
|
3760
|
+
}
|
|
3761
|
+
|
|
3762
|
+
if (private_data->bitmap.buffer.data != NULL) {
|
|
3763
|
+
NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1));
|
|
3764
|
+
}
|
|
3765
|
+
|
|
3766
|
+
array->length++;
|
|
3767
|
+
return NANOARROW_OK;
|
|
3768
|
+
}
|
|
3769
|
+
|
|
3770
|
+
static inline ArrowErrorCode ArrowArrayAppendString(struct ArrowArray* array,
|
|
3771
|
+
struct ArrowStringView value) {
|
|
3772
|
+
struct ArrowArrayPrivateData* private_data =
|
|
3773
|
+
(struct ArrowArrayPrivateData*)array->private_data;
|
|
3774
|
+
|
|
3775
|
+
struct ArrowBufferView buffer_view;
|
|
3776
|
+
buffer_view.data.data = value.data;
|
|
3777
|
+
buffer_view.size_bytes = value.size_bytes;
|
|
3778
|
+
|
|
3779
|
+
switch (private_data->storage_type) {
|
|
3780
|
+
case NANOARROW_TYPE_STRING:
|
|
3781
|
+
case NANOARROW_TYPE_LARGE_STRING:
|
|
3782
|
+
case NANOARROW_TYPE_STRING_VIEW:
|
|
3783
|
+
case NANOARROW_TYPE_BINARY:
|
|
3784
|
+
case NANOARROW_TYPE_LARGE_BINARY:
|
|
3785
|
+
case NANOARROW_TYPE_BINARY_VIEW:
|
|
3786
|
+
return ArrowArrayAppendBytes(array, buffer_view);
|
|
3787
|
+
default:
|
|
3788
|
+
return EINVAL;
|
|
3789
|
+
}
|
|
3790
|
+
}
|
|
3791
|
+
|
|
3792
|
+
static inline ArrowErrorCode ArrowArrayAppendInterval(struct ArrowArray* array,
|
|
3793
|
+
const struct ArrowInterval* value) {
|
|
3794
|
+
struct ArrowArrayPrivateData* private_data =
|
|
3795
|
+
(struct ArrowArrayPrivateData*)array->private_data;
|
|
3796
|
+
|
|
3797
|
+
struct ArrowBuffer* data_buffer = ArrowArrayBuffer(array, 1);
|
|
3798
|
+
|
|
3799
|
+
switch (private_data->storage_type) {
|
|
3800
|
+
case NANOARROW_TYPE_INTERVAL_MONTHS: {
|
|
3801
|
+
if (value->type != NANOARROW_TYPE_INTERVAL_MONTHS) {
|
|
3802
|
+
return EINVAL;
|
|
3803
|
+
}
|
|
3804
|
+
|
|
3805
|
+
NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(data_buffer, value->months));
|
|
3806
|
+
break;
|
|
3807
|
+
}
|
|
3808
|
+
case NANOARROW_TYPE_INTERVAL_DAY_TIME: {
|
|
3809
|
+
if (value->type != NANOARROW_TYPE_INTERVAL_DAY_TIME) {
|
|
3810
|
+
return EINVAL;
|
|
3811
|
+
}
|
|
3812
|
+
|
|
3813
|
+
NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(data_buffer, value->days));
|
|
3814
|
+
NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(data_buffer, value->ms));
|
|
3815
|
+
break;
|
|
3816
|
+
}
|
|
3817
|
+
case NANOARROW_TYPE_INTERVAL_MONTH_DAY_NANO: {
|
|
3818
|
+
if (value->type != NANOARROW_TYPE_INTERVAL_MONTH_DAY_NANO) {
|
|
3819
|
+
return EINVAL;
|
|
3820
|
+
}
|
|
3821
|
+
|
|
3822
|
+
NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(data_buffer, value->months));
|
|
3823
|
+
NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(data_buffer, value->days));
|
|
3824
|
+
NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt64(data_buffer, value->ns));
|
|
3825
|
+
break;
|
|
3826
|
+
}
|
|
3827
|
+
default:
|
|
3828
|
+
return EINVAL;
|
|
3829
|
+
}
|
|
3830
|
+
|
|
3831
|
+
if (private_data->bitmap.buffer.data != NULL) {
|
|
3832
|
+
NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1));
|
|
3833
|
+
}
|
|
3834
|
+
|
|
3835
|
+
array->length++;
|
|
3836
|
+
return NANOARROW_OK;
|
|
3837
|
+
}
|
|
3838
|
+
|
|
3839
|
+
static inline ArrowErrorCode ArrowArrayAppendDecimal(struct ArrowArray* array,
|
|
3840
|
+
const struct ArrowDecimal* value) {
|
|
3841
|
+
struct ArrowArrayPrivateData* private_data =
|
|
3842
|
+
(struct ArrowArrayPrivateData*)array->private_data;
|
|
3843
|
+
struct ArrowBuffer* data_buffer = ArrowArrayBuffer(array, 1);
|
|
3844
|
+
|
|
3845
|
+
switch (private_data->storage_type) {
|
|
3846
|
+
case NANOARROW_TYPE_DECIMAL32:
|
|
3847
|
+
if (value->n_words != 0) {
|
|
3848
|
+
return EINVAL;
|
|
3849
|
+
} else {
|
|
3850
|
+
NANOARROW_RETURN_NOT_OK(
|
|
3851
|
+
ArrowBufferAppend(data_buffer, value->words, sizeof(uint32_t)));
|
|
3852
|
+
break;
|
|
3853
|
+
}
|
|
3854
|
+
case NANOARROW_TYPE_DECIMAL64:
|
|
3855
|
+
if (value->n_words != 1) {
|
|
3856
|
+
return EINVAL;
|
|
3857
|
+
} else {
|
|
3858
|
+
NANOARROW_RETURN_NOT_OK(
|
|
3859
|
+
ArrowBufferAppend(data_buffer, value->words, sizeof(uint64_t)));
|
|
3860
|
+
break;
|
|
3861
|
+
}
|
|
3862
|
+
case NANOARROW_TYPE_DECIMAL128:
|
|
3863
|
+
if (value->n_words != 2) {
|
|
3864
|
+
return EINVAL;
|
|
3865
|
+
} else {
|
|
3866
|
+
NANOARROW_RETURN_NOT_OK(
|
|
3867
|
+
ArrowBufferAppend(data_buffer, value->words, 2 * sizeof(uint64_t)));
|
|
3868
|
+
break;
|
|
3869
|
+
}
|
|
3870
|
+
case NANOARROW_TYPE_DECIMAL256:
|
|
3871
|
+
if (value->n_words != 4) {
|
|
3872
|
+
return EINVAL;
|
|
3873
|
+
} else {
|
|
3874
|
+
NANOARROW_RETURN_NOT_OK(
|
|
3875
|
+
ArrowBufferAppend(data_buffer, value->words, 4 * sizeof(uint64_t)));
|
|
3876
|
+
break;
|
|
3877
|
+
}
|
|
3878
|
+
default:
|
|
3879
|
+
return EINVAL;
|
|
3880
|
+
}
|
|
3881
|
+
|
|
3882
|
+
if (private_data->bitmap.buffer.data != NULL) {
|
|
3883
|
+
NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1));
|
|
3884
|
+
}
|
|
3885
|
+
|
|
3886
|
+
array->length++;
|
|
3887
|
+
return NANOARROW_OK;
|
|
3888
|
+
}
|
|
3889
|
+
|
|
3890
|
+
static inline ArrowErrorCode ArrowArrayFinishElement(struct ArrowArray* array) {
|
|
3891
|
+
struct ArrowArrayPrivateData* private_data =
|
|
3892
|
+
(struct ArrowArrayPrivateData*)array->private_data;
|
|
3893
|
+
|
|
3894
|
+
int64_t child_length;
|
|
3895
|
+
|
|
3896
|
+
switch (private_data->storage_type) {
|
|
3897
|
+
case NANOARROW_TYPE_LIST:
|
|
3898
|
+
case NANOARROW_TYPE_MAP:
|
|
3899
|
+
child_length = array->children[0]->length;
|
|
3900
|
+
if (child_length > INT32_MAX) {
|
|
3901
|
+
return EOVERFLOW;
|
|
3902
|
+
}
|
|
3903
|
+
|
|
3904
|
+
NANOARROW_RETURN_NOT_OK(
|
|
3905
|
+
ArrowBufferAppendInt32(ArrowArrayBuffer(array, 1), (int32_t)child_length));
|
|
3906
|
+
break;
|
|
3907
|
+
case NANOARROW_TYPE_LARGE_LIST:
|
|
3908
|
+
child_length = array->children[0]->length;
|
|
3909
|
+
NANOARROW_RETURN_NOT_OK(
|
|
3910
|
+
ArrowBufferAppendInt64(ArrowArrayBuffer(array, 1), child_length));
|
|
3911
|
+
break;
|
|
3912
|
+
case NANOARROW_TYPE_FIXED_SIZE_LIST:
|
|
3913
|
+
child_length = array->children[0]->length;
|
|
3914
|
+
if (child_length !=
|
|
3915
|
+
((array->length + 1) * private_data->layout.child_size_elements)) {
|
|
3916
|
+
return EINVAL;
|
|
3917
|
+
}
|
|
3918
|
+
break;
|
|
3919
|
+
case NANOARROW_TYPE_LIST_VIEW: {
|
|
3920
|
+
child_length = array->children[0]->length;
|
|
3921
|
+
if (child_length > INT32_MAX) {
|
|
3922
|
+
return EOVERFLOW;
|
|
3923
|
+
}
|
|
3924
|
+
|
|
3925
|
+
const int32_t last_valid_offset = (int32_t)private_data->list_view_offset;
|
|
3926
|
+
NANOARROW_RETURN_NOT_OK(
|
|
3927
|
+
ArrowBufferAppendInt32(ArrowArrayBuffer(array, 1), last_valid_offset));
|
|
3928
|
+
NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(
|
|
3929
|
+
ArrowArrayBuffer(array, 2), (int32_t)child_length - last_valid_offset));
|
|
3930
|
+
private_data->list_view_offset = child_length;
|
|
3931
|
+
break;
|
|
3932
|
+
}
|
|
3933
|
+
case NANOARROW_TYPE_LARGE_LIST_VIEW: {
|
|
3934
|
+
child_length = array->children[0]->length;
|
|
3935
|
+
const int64_t last_valid_offset = private_data->list_view_offset;
|
|
3936
|
+
NANOARROW_RETURN_NOT_OK(
|
|
3937
|
+
ArrowBufferAppendInt64(ArrowArrayBuffer(array, 1), last_valid_offset));
|
|
3938
|
+
NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt64(ArrowArrayBuffer(array, 2),
|
|
3939
|
+
child_length - last_valid_offset));
|
|
3940
|
+
private_data->list_view_offset = child_length;
|
|
3941
|
+
break;
|
|
3942
|
+
}
|
|
3943
|
+
|
|
3944
|
+
case NANOARROW_TYPE_STRUCT:
|
|
3945
|
+
for (int64_t i = 0; i < array->n_children; i++) {
|
|
3946
|
+
child_length = array->children[i]->length;
|
|
3947
|
+
if (child_length != (array->length + 1)) {
|
|
3948
|
+
return EINVAL;
|
|
3949
|
+
}
|
|
3950
|
+
}
|
|
3951
|
+
break;
|
|
3952
|
+
default:
|
|
3953
|
+
return EINVAL;
|
|
3954
|
+
}
|
|
3955
|
+
|
|
3956
|
+
if (private_data->bitmap.buffer.data != NULL) {
|
|
3957
|
+
NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1));
|
|
3958
|
+
}
|
|
3959
|
+
|
|
3960
|
+
array->length++;
|
|
3961
|
+
return NANOARROW_OK;
|
|
3962
|
+
}
|
|
3963
|
+
|
|
3964
|
+
static inline ArrowErrorCode ArrowArrayFinishUnionElement(struct ArrowArray* array,
|
|
3965
|
+
int8_t type_id) {
|
|
3966
|
+
struct ArrowArrayPrivateData* private_data =
|
|
3967
|
+
(struct ArrowArrayPrivateData*)array->private_data;
|
|
3968
|
+
|
|
3969
|
+
int64_t child_index = _ArrowArrayUnionChildIndex(array, type_id);
|
|
3970
|
+
if (child_index < 0 || child_index >= array->n_children) {
|
|
3971
|
+
return EINVAL;
|
|
3972
|
+
}
|
|
3973
|
+
|
|
3974
|
+
switch (private_data->storage_type) {
|
|
3975
|
+
case NANOARROW_TYPE_DENSE_UNION:
|
|
3976
|
+
// Append the target child length to the union offsets buffer
|
|
3977
|
+
_NANOARROW_CHECK_RANGE(array->children[child_index]->length, 0, INT32_MAX);
|
|
3978
|
+
NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(
|
|
3979
|
+
ArrowArrayBuffer(array, 1), (int32_t)array->children[child_index]->length - 1));
|
|
3980
|
+
break;
|
|
3981
|
+
case NANOARROW_TYPE_SPARSE_UNION:
|
|
3982
|
+
// Append one empty to any non-target column that isn't already the right length
|
|
3983
|
+
// or abort if appending a null will result in a column with invalid length
|
|
3984
|
+
for (int64_t i = 0; i < array->n_children; i++) {
|
|
3985
|
+
if (i == child_index || array->children[i]->length == (array->length + 1)) {
|
|
3986
|
+
continue;
|
|
3987
|
+
}
|
|
3988
|
+
|
|
3989
|
+
if (array->children[i]->length != array->length) {
|
|
3990
|
+
return EINVAL;
|
|
3991
|
+
}
|
|
3992
|
+
|
|
3993
|
+
NANOARROW_RETURN_NOT_OK(ArrowArrayAppendEmpty(array->children[i], 1));
|
|
3994
|
+
}
|
|
3995
|
+
|
|
3996
|
+
break;
|
|
3997
|
+
default:
|
|
3998
|
+
return EINVAL;
|
|
3999
|
+
}
|
|
4000
|
+
|
|
4001
|
+
// Write to the type_ids buffer
|
|
4002
|
+
NANOARROW_RETURN_NOT_OK(
|
|
4003
|
+
ArrowBufferAppendInt8(ArrowArrayBuffer(array, 0), (int8_t)type_id));
|
|
4004
|
+
array->length++;
|
|
4005
|
+
return NANOARROW_OK;
|
|
4006
|
+
}
|
|
4007
|
+
|
|
4008
|
+
static inline void ArrowArrayViewMove(struct ArrowArrayView* src,
|
|
4009
|
+
struct ArrowArrayView* dst) {
|
|
4010
|
+
memcpy(dst, src, sizeof(struct ArrowArrayView));
|
|
4011
|
+
ArrowArrayViewInitFromType(src, NANOARROW_TYPE_UNINITIALIZED);
|
|
4012
|
+
}
|
|
4013
|
+
|
|
4014
|
+
static inline int64_t ArrowArrayViewGetNumBuffers(struct ArrowArrayView* array_view) {
|
|
4015
|
+
switch (array_view->storage_type) {
|
|
4016
|
+
case NANOARROW_TYPE_BINARY_VIEW:
|
|
4017
|
+
case NANOARROW_TYPE_STRING_VIEW:
|
|
4018
|
+
return NANOARROW_BINARY_VIEW_FIXED_BUFFERS + array_view->n_variadic_buffers + 1;
|
|
4019
|
+
default:
|
|
4020
|
+
break;
|
|
4021
|
+
}
|
|
4022
|
+
|
|
4023
|
+
int64_t n_buffers = 0;
|
|
4024
|
+
for (int i = 0; i < NANOARROW_MAX_FIXED_BUFFERS; i++) {
|
|
4025
|
+
if (array_view->layout.buffer_type[i] == NANOARROW_BUFFER_TYPE_NONE) {
|
|
4026
|
+
break;
|
|
4027
|
+
}
|
|
4028
|
+
|
|
4029
|
+
n_buffers++;
|
|
4030
|
+
}
|
|
4031
|
+
|
|
4032
|
+
return n_buffers;
|
|
4033
|
+
}
|
|
4034
|
+
|
|
4035
|
+
static inline struct ArrowBufferView ArrowArrayViewGetBufferView(
|
|
4036
|
+
struct ArrowArrayView* array_view, int64_t i) {
|
|
4037
|
+
switch (array_view->storage_type) {
|
|
4038
|
+
case NANOARROW_TYPE_BINARY_VIEW:
|
|
4039
|
+
case NANOARROW_TYPE_STRING_VIEW:
|
|
4040
|
+
if (i < NANOARROW_BINARY_VIEW_FIXED_BUFFERS) {
|
|
4041
|
+
return array_view->buffer_views[i];
|
|
4042
|
+
} else if (i >=
|
|
4043
|
+
(array_view->n_variadic_buffers + NANOARROW_BINARY_VIEW_FIXED_BUFFERS)) {
|
|
4044
|
+
struct ArrowBufferView view;
|
|
4045
|
+
view.data.as_int64 = array_view->variadic_buffer_sizes;
|
|
4046
|
+
view.size_bytes = array_view->n_variadic_buffers * sizeof(double);
|
|
4047
|
+
return view;
|
|
4048
|
+
} else {
|
|
4049
|
+
struct ArrowBufferView view;
|
|
4050
|
+
view.data.data =
|
|
4051
|
+
array_view->variadic_buffers[i - NANOARROW_BINARY_VIEW_FIXED_BUFFERS];
|
|
4052
|
+
view.size_bytes =
|
|
4053
|
+
array_view->variadic_buffer_sizes[i - NANOARROW_BINARY_VIEW_FIXED_BUFFERS];
|
|
4054
|
+
return view;
|
|
4055
|
+
}
|
|
4056
|
+
default:
|
|
4057
|
+
// We need this check to avoid -Warray-bounds from complaining
|
|
4058
|
+
if (i >= NANOARROW_MAX_FIXED_BUFFERS) {
|
|
4059
|
+
struct ArrowBufferView view;
|
|
4060
|
+
view.data.data = NULL;
|
|
4061
|
+
view.size_bytes = 0;
|
|
4062
|
+
return view;
|
|
4063
|
+
} else {
|
|
4064
|
+
return array_view->buffer_views[i];
|
|
4065
|
+
}
|
|
4066
|
+
}
|
|
4067
|
+
}
|
|
4068
|
+
|
|
4069
|
+
enum ArrowBufferType ArrowArrayViewGetBufferType(struct ArrowArrayView* array_view,
|
|
4070
|
+
int64_t i) {
|
|
4071
|
+
switch (array_view->storage_type) {
|
|
4072
|
+
case NANOARROW_TYPE_BINARY_VIEW:
|
|
4073
|
+
case NANOARROW_TYPE_STRING_VIEW:
|
|
4074
|
+
if (i < NANOARROW_BINARY_VIEW_FIXED_BUFFERS) {
|
|
4075
|
+
return array_view->layout.buffer_type[i];
|
|
4076
|
+
} else if (i ==
|
|
4077
|
+
(array_view->n_variadic_buffers + NANOARROW_BINARY_VIEW_FIXED_BUFFERS)) {
|
|
4078
|
+
return NANOARROW_BUFFER_TYPE_VARIADIC_SIZE;
|
|
4079
|
+
} else {
|
|
4080
|
+
return NANOARROW_BUFFER_TYPE_VARIADIC_DATA;
|
|
4081
|
+
}
|
|
4082
|
+
default:
|
|
4083
|
+
// We need this check to avoid -Warray-bounds from complaining
|
|
4084
|
+
if (i >= NANOARROW_MAX_FIXED_BUFFERS) {
|
|
4085
|
+
return NANOARROW_BUFFER_TYPE_NONE;
|
|
4086
|
+
} else {
|
|
4087
|
+
return array_view->layout.buffer_type[i];
|
|
4088
|
+
}
|
|
4089
|
+
}
|
|
4090
|
+
}
|
|
4091
|
+
|
|
4092
|
+
static inline enum ArrowType ArrowArrayViewGetBufferDataType(
|
|
4093
|
+
struct ArrowArrayView* array_view, int64_t i) {
|
|
4094
|
+
switch (array_view->storage_type) {
|
|
4095
|
+
case NANOARROW_TYPE_BINARY_VIEW:
|
|
4096
|
+
case NANOARROW_TYPE_STRING_VIEW:
|
|
4097
|
+
if (i < NANOARROW_BINARY_VIEW_FIXED_BUFFERS) {
|
|
4098
|
+
return array_view->layout.buffer_data_type[i];
|
|
4099
|
+
} else if (i >=
|
|
4100
|
+
(array_view->n_variadic_buffers + NANOARROW_BINARY_VIEW_FIXED_BUFFERS)) {
|
|
4101
|
+
return NANOARROW_TYPE_INT64;
|
|
4102
|
+
} else if (array_view->storage_type == NANOARROW_TYPE_BINARY_VIEW) {
|
|
4103
|
+
return NANOARROW_TYPE_BINARY;
|
|
4104
|
+
} else {
|
|
4105
|
+
return NANOARROW_TYPE_STRING;
|
|
4106
|
+
}
|
|
4107
|
+
default:
|
|
4108
|
+
// We need this check to avoid -Warray-bounds from complaining
|
|
4109
|
+
if (i >= NANOARROW_MAX_FIXED_BUFFERS) {
|
|
4110
|
+
return NANOARROW_TYPE_UNINITIALIZED;
|
|
4111
|
+
} else {
|
|
4112
|
+
return array_view->layout.buffer_data_type[i];
|
|
4113
|
+
}
|
|
4114
|
+
}
|
|
4115
|
+
}
|
|
4116
|
+
|
|
4117
|
+
static inline int64_t ArrowArrayViewGetBufferElementSizeBits(
|
|
4118
|
+
struct ArrowArrayView* array_view, int64_t i) {
|
|
4119
|
+
switch (array_view->storage_type) {
|
|
4120
|
+
case NANOARROW_TYPE_BINARY_VIEW:
|
|
4121
|
+
case NANOARROW_TYPE_STRING_VIEW:
|
|
4122
|
+
if (i < NANOARROW_BINARY_VIEW_FIXED_BUFFERS) {
|
|
4123
|
+
return array_view->layout.element_size_bits[i];
|
|
4124
|
+
} else if (i >=
|
|
4125
|
+
(array_view->n_variadic_buffers + NANOARROW_BINARY_VIEW_FIXED_BUFFERS)) {
|
|
4126
|
+
return sizeof(int64_t) * 8;
|
|
4127
|
+
} else {
|
|
4128
|
+
return 0;
|
|
4129
|
+
}
|
|
4130
|
+
default:
|
|
4131
|
+
// We need this check to avoid -Warray-bounds from complaining
|
|
4132
|
+
if (i >= NANOARROW_MAX_FIXED_BUFFERS) {
|
|
4133
|
+
return 0;
|
|
4134
|
+
} else {
|
|
4135
|
+
return array_view->layout.element_size_bits[i];
|
|
4136
|
+
}
|
|
4137
|
+
}
|
|
4138
|
+
}
|
|
4139
|
+
|
|
4140
|
+
static inline int8_t ArrowArrayViewIsNull(const struct ArrowArrayView* array_view,
|
|
4141
|
+
int64_t i) {
|
|
4142
|
+
const uint8_t* validity_buffer = array_view->buffer_views[0].data.as_uint8;
|
|
4143
|
+
i += array_view->offset;
|
|
4144
|
+
switch (array_view->storage_type) {
|
|
4145
|
+
case NANOARROW_TYPE_NA:
|
|
4146
|
+
return 0x01;
|
|
4147
|
+
case NANOARROW_TYPE_DENSE_UNION:
|
|
4148
|
+
case NANOARROW_TYPE_SPARSE_UNION:
|
|
4149
|
+
// Unions are "never null" in Arrow land
|
|
4150
|
+
return 0x00;
|
|
4151
|
+
default:
|
|
4152
|
+
return validity_buffer != NULL && !ArrowBitGet(validity_buffer, i);
|
|
4153
|
+
}
|
|
4154
|
+
}
|
|
4155
|
+
|
|
4156
|
+
static inline int64_t ArrowArrayViewComputeNullCount(
|
|
4157
|
+
const struct ArrowArrayView* array_view) {
|
|
4158
|
+
if (array_view->length == 0) {
|
|
4159
|
+
return 0;
|
|
4160
|
+
}
|
|
4161
|
+
|
|
4162
|
+
switch (array_view->storage_type) {
|
|
4163
|
+
case NANOARROW_TYPE_NA:
|
|
4164
|
+
return array_view->length;
|
|
4165
|
+
case NANOARROW_TYPE_DENSE_UNION:
|
|
4166
|
+
case NANOARROW_TYPE_SPARSE_UNION:
|
|
4167
|
+
// Unions are "never null" in Arrow land
|
|
4168
|
+
return 0;
|
|
4169
|
+
default:
|
|
4170
|
+
break;
|
|
4171
|
+
}
|
|
4172
|
+
|
|
4173
|
+
const uint8_t* validity_buffer = array_view->buffer_views[0].data.as_uint8;
|
|
4174
|
+
if (validity_buffer == NULL) {
|
|
4175
|
+
return 0;
|
|
4176
|
+
}
|
|
4177
|
+
return array_view->length -
|
|
4178
|
+
ArrowBitCountSet(validity_buffer, array_view->offset, array_view->length);
|
|
4179
|
+
}
|
|
4180
|
+
|
|
4181
|
+
static inline int8_t ArrowArrayViewUnionTypeId(const struct ArrowArrayView* array_view,
|
|
4182
|
+
int64_t i) {
|
|
4183
|
+
switch (array_view->storage_type) {
|
|
4184
|
+
case NANOARROW_TYPE_DENSE_UNION:
|
|
4185
|
+
case NANOARROW_TYPE_SPARSE_UNION:
|
|
4186
|
+
return array_view->buffer_views[0].data.as_int8[array_view->offset + i];
|
|
4187
|
+
default:
|
|
4188
|
+
return -1;
|
|
4189
|
+
}
|
|
4190
|
+
}
|
|
4191
|
+
|
|
4192
|
+
static inline int8_t ArrowArrayViewUnionChildIndex(
|
|
4193
|
+
const struct ArrowArrayView* array_view, int64_t i) {
|
|
4194
|
+
int8_t type_id = ArrowArrayViewUnionTypeId(array_view, i);
|
|
4195
|
+
if (array_view->union_type_id_map == NULL) {
|
|
4196
|
+
return type_id;
|
|
4197
|
+
} else {
|
|
4198
|
+
return array_view->union_type_id_map[type_id];
|
|
4199
|
+
}
|
|
4200
|
+
}
|
|
4201
|
+
|
|
4202
|
+
static inline int64_t ArrowArrayViewUnionChildOffset(
|
|
4203
|
+
const struct ArrowArrayView* array_view, int64_t i) {
|
|
4204
|
+
switch (array_view->storage_type) {
|
|
4205
|
+
case NANOARROW_TYPE_DENSE_UNION:
|
|
4206
|
+
return array_view->buffer_views[1].data.as_int32[array_view->offset + i];
|
|
4207
|
+
case NANOARROW_TYPE_SPARSE_UNION:
|
|
4208
|
+
return array_view->offset + i;
|
|
4209
|
+
default:
|
|
4210
|
+
return -1;
|
|
4211
|
+
}
|
|
4212
|
+
}
|
|
4213
|
+
|
|
4214
|
+
static inline int64_t ArrowArrayViewListChildOffset(
|
|
4215
|
+
const struct ArrowArrayView* array_view, int64_t i) {
|
|
4216
|
+
switch (array_view->storage_type) {
|
|
4217
|
+
case NANOARROW_TYPE_LIST:
|
|
4218
|
+
case NANOARROW_TYPE_MAP:
|
|
4219
|
+
case NANOARROW_TYPE_LIST_VIEW:
|
|
4220
|
+
return array_view->buffer_views[1].data.as_int32[i];
|
|
4221
|
+
case NANOARROW_TYPE_LARGE_LIST:
|
|
4222
|
+
case NANOARROW_TYPE_LARGE_LIST_VIEW:
|
|
4223
|
+
return array_view->buffer_views[1].data.as_int64[i];
|
|
4224
|
+
default:
|
|
4225
|
+
return -1;
|
|
4226
|
+
}
|
|
4227
|
+
}
|
|
4228
|
+
|
|
4229
|
+
static struct ArrowBufferView ArrowArrayViewGetBytesFromViewArrayUnsafe(
|
|
4230
|
+
const struct ArrowArrayView* array_view, int64_t i) {
|
|
4231
|
+
const union ArrowBinaryView* bv = &array_view->buffer_views[1].data.as_binary_view[i];
|
|
4232
|
+
struct ArrowBufferView out = {{NULL}, bv->inlined.size};
|
|
4233
|
+
if (bv->inlined.size <= NANOARROW_BINARY_VIEW_INLINE_SIZE) {
|
|
4234
|
+
out.data.as_uint8 = bv->inlined.data;
|
|
4235
|
+
return out;
|
|
4236
|
+
}
|
|
4237
|
+
|
|
4238
|
+
out.data.data = array_view->variadic_buffers[bv->ref.buffer_index];
|
|
4239
|
+
out.data.as_uint8 += bv->ref.offset;
|
|
4240
|
+
return out;
|
|
4241
|
+
}
|
|
4242
|
+
|
|
4243
|
+
static inline int64_t ArrowArrayViewGetIntUnsafe(const struct ArrowArrayView* array_view,
|
|
4244
|
+
int64_t i) {
|
|
4245
|
+
const struct ArrowBufferView* data_view = &array_view->buffer_views[1];
|
|
4246
|
+
i += array_view->offset;
|
|
4247
|
+
switch (array_view->storage_type) {
|
|
4248
|
+
case NANOARROW_TYPE_INT64:
|
|
4249
|
+
return data_view->data.as_int64[i];
|
|
4250
|
+
case NANOARROW_TYPE_UINT64:
|
|
4251
|
+
return data_view->data.as_uint64[i];
|
|
4252
|
+
case NANOARROW_TYPE_INTERVAL_MONTHS:
|
|
4253
|
+
case NANOARROW_TYPE_INT32:
|
|
4254
|
+
return data_view->data.as_int32[i];
|
|
4255
|
+
case NANOARROW_TYPE_UINT32:
|
|
4256
|
+
return data_view->data.as_uint32[i];
|
|
4257
|
+
case NANOARROW_TYPE_INT16:
|
|
4258
|
+
return data_view->data.as_int16[i];
|
|
4259
|
+
case NANOARROW_TYPE_UINT16:
|
|
4260
|
+
return data_view->data.as_uint16[i];
|
|
4261
|
+
case NANOARROW_TYPE_INT8:
|
|
4262
|
+
return data_view->data.as_int8[i];
|
|
4263
|
+
case NANOARROW_TYPE_UINT8:
|
|
4264
|
+
return data_view->data.as_uint8[i];
|
|
4265
|
+
case NANOARROW_TYPE_DOUBLE:
|
|
4266
|
+
return (int64_t)data_view->data.as_double[i];
|
|
4267
|
+
case NANOARROW_TYPE_FLOAT:
|
|
4268
|
+
return (int64_t)data_view->data.as_float[i];
|
|
4269
|
+
case NANOARROW_TYPE_HALF_FLOAT:
|
|
4270
|
+
return (int64_t)ArrowHalfFloatToFloat(data_view->data.as_uint16[i]);
|
|
4271
|
+
case NANOARROW_TYPE_BOOL:
|
|
4272
|
+
return ArrowBitGet(data_view->data.as_uint8, i);
|
|
4273
|
+
default:
|
|
4274
|
+
return INT64_MAX;
|
|
4275
|
+
}
|
|
4276
|
+
}
|
|
4277
|
+
|
|
4278
|
+
static inline uint64_t ArrowArrayViewGetUIntUnsafe(
|
|
4279
|
+
const struct ArrowArrayView* array_view, int64_t i) {
|
|
4280
|
+
i += array_view->offset;
|
|
4281
|
+
const struct ArrowBufferView* data_view = &array_view->buffer_views[1];
|
|
4282
|
+
switch (array_view->storage_type) {
|
|
4283
|
+
case NANOARROW_TYPE_INT64:
|
|
4284
|
+
return data_view->data.as_int64[i];
|
|
4285
|
+
case NANOARROW_TYPE_UINT64:
|
|
4286
|
+
return data_view->data.as_uint64[i];
|
|
4287
|
+
case NANOARROW_TYPE_INTERVAL_MONTHS:
|
|
4288
|
+
case NANOARROW_TYPE_INT32:
|
|
4289
|
+
return data_view->data.as_int32[i];
|
|
4290
|
+
case NANOARROW_TYPE_UINT32:
|
|
4291
|
+
return data_view->data.as_uint32[i];
|
|
4292
|
+
case NANOARROW_TYPE_INT16:
|
|
4293
|
+
return data_view->data.as_int16[i];
|
|
4294
|
+
case NANOARROW_TYPE_UINT16:
|
|
4295
|
+
return data_view->data.as_uint16[i];
|
|
4296
|
+
case NANOARROW_TYPE_INT8:
|
|
4297
|
+
return data_view->data.as_int8[i];
|
|
4298
|
+
case NANOARROW_TYPE_UINT8:
|
|
4299
|
+
return data_view->data.as_uint8[i];
|
|
4300
|
+
case NANOARROW_TYPE_DOUBLE:
|
|
4301
|
+
return (uint64_t)data_view->data.as_double[i];
|
|
4302
|
+
case NANOARROW_TYPE_FLOAT:
|
|
4303
|
+
return (uint64_t)data_view->data.as_float[i];
|
|
4304
|
+
case NANOARROW_TYPE_HALF_FLOAT:
|
|
4305
|
+
return (uint64_t)ArrowHalfFloatToFloat(data_view->data.as_uint16[i]);
|
|
4306
|
+
case NANOARROW_TYPE_BOOL:
|
|
4307
|
+
return ArrowBitGet(data_view->data.as_uint8, i);
|
|
4308
|
+
default:
|
|
4309
|
+
return UINT64_MAX;
|
|
4310
|
+
}
|
|
4311
|
+
}
|
|
4312
|
+
|
|
4313
|
+
static inline double ArrowArrayViewGetDoubleUnsafe(
|
|
4314
|
+
const struct ArrowArrayView* array_view, int64_t i) {
|
|
4315
|
+
i += array_view->offset;
|
|
4316
|
+
const struct ArrowBufferView* data_view = &array_view->buffer_views[1];
|
|
4317
|
+
switch (array_view->storage_type) {
|
|
4318
|
+
case NANOARROW_TYPE_INT64:
|
|
4319
|
+
return (double)data_view->data.as_int64[i];
|
|
4320
|
+
case NANOARROW_TYPE_UINT64:
|
|
4321
|
+
return (double)data_view->data.as_uint64[i];
|
|
4322
|
+
case NANOARROW_TYPE_INT32:
|
|
4323
|
+
return data_view->data.as_int32[i];
|
|
4324
|
+
case NANOARROW_TYPE_UINT32:
|
|
4325
|
+
return data_view->data.as_uint32[i];
|
|
4326
|
+
case NANOARROW_TYPE_INT16:
|
|
4327
|
+
return data_view->data.as_int16[i];
|
|
4328
|
+
case NANOARROW_TYPE_UINT16:
|
|
4329
|
+
return data_view->data.as_uint16[i];
|
|
4330
|
+
case NANOARROW_TYPE_INT8:
|
|
4331
|
+
return data_view->data.as_int8[i];
|
|
4332
|
+
case NANOARROW_TYPE_UINT8:
|
|
4333
|
+
return data_view->data.as_uint8[i];
|
|
4334
|
+
case NANOARROW_TYPE_DOUBLE:
|
|
4335
|
+
return data_view->data.as_double[i];
|
|
4336
|
+
case NANOARROW_TYPE_FLOAT:
|
|
4337
|
+
return data_view->data.as_float[i];
|
|
4338
|
+
case NANOARROW_TYPE_HALF_FLOAT:
|
|
4339
|
+
return ArrowHalfFloatToFloat(data_view->data.as_uint16[i]);
|
|
4340
|
+
case NANOARROW_TYPE_BOOL:
|
|
4341
|
+
return ArrowBitGet(data_view->data.as_uint8, i);
|
|
4342
|
+
default:
|
|
4343
|
+
return DBL_MAX;
|
|
4344
|
+
}
|
|
4345
|
+
}
|
|
4346
|
+
|
|
4347
|
+
static inline struct ArrowStringView ArrowArrayViewGetStringUnsafe(
|
|
4348
|
+
const struct ArrowArrayView* array_view, int64_t i) {
|
|
4349
|
+
i += array_view->offset;
|
|
4350
|
+
const struct ArrowBufferView* offsets_view = &array_view->buffer_views[1];
|
|
4351
|
+
const char* data_view = array_view->buffer_views[2].data.as_char;
|
|
4352
|
+
|
|
4353
|
+
struct ArrowStringView view;
|
|
4354
|
+
switch (array_view->storage_type) {
|
|
4355
|
+
case NANOARROW_TYPE_STRING:
|
|
4356
|
+
case NANOARROW_TYPE_BINARY:
|
|
4357
|
+
view.data = data_view + offsets_view->data.as_int32[i];
|
|
4358
|
+
view.size_bytes =
|
|
4359
|
+
(int64_t)offsets_view->data.as_int32[i + 1] - offsets_view->data.as_int32[i];
|
|
4360
|
+
break;
|
|
4361
|
+
case NANOARROW_TYPE_LARGE_STRING:
|
|
4362
|
+
case NANOARROW_TYPE_LARGE_BINARY:
|
|
4363
|
+
view.data = data_view + offsets_view->data.as_int64[i];
|
|
4364
|
+
view.size_bytes =
|
|
4365
|
+
offsets_view->data.as_int64[i + 1] - offsets_view->data.as_int64[i];
|
|
4366
|
+
break;
|
|
4367
|
+
case NANOARROW_TYPE_FIXED_SIZE_BINARY:
|
|
4368
|
+
view.size_bytes = array_view->layout.element_size_bits[1] / 8;
|
|
4369
|
+
view.data = array_view->buffer_views[1].data.as_char + (i * view.size_bytes);
|
|
4370
|
+
break;
|
|
4371
|
+
case NANOARROW_TYPE_STRING_VIEW:
|
|
4372
|
+
case NANOARROW_TYPE_BINARY_VIEW: {
|
|
4373
|
+
struct ArrowBufferView buf_view =
|
|
4374
|
+
ArrowArrayViewGetBytesFromViewArrayUnsafe(array_view, i);
|
|
4375
|
+
view.data = buf_view.data.as_char;
|
|
4376
|
+
view.size_bytes = buf_view.size_bytes;
|
|
4377
|
+
break;
|
|
4378
|
+
}
|
|
4379
|
+
default:
|
|
4380
|
+
view.data = NULL;
|
|
4381
|
+
view.size_bytes = 0;
|
|
4382
|
+
break;
|
|
4383
|
+
}
|
|
4384
|
+
|
|
4385
|
+
return view;
|
|
4386
|
+
}
|
|
4387
|
+
|
|
4388
|
+
static inline struct ArrowBufferView ArrowArrayViewGetBytesUnsafe(
|
|
4389
|
+
const struct ArrowArrayView* array_view, int64_t i) {
|
|
4390
|
+
i += array_view->offset;
|
|
4391
|
+
const struct ArrowBufferView* offsets_view = &array_view->buffer_views[1];
|
|
4392
|
+
const uint8_t* data_view = array_view->buffer_views[2].data.as_uint8;
|
|
4393
|
+
|
|
4394
|
+
struct ArrowBufferView view;
|
|
4395
|
+
switch (array_view->storage_type) {
|
|
4396
|
+
case NANOARROW_TYPE_STRING:
|
|
4397
|
+
case NANOARROW_TYPE_BINARY:
|
|
4398
|
+
view.size_bytes =
|
|
4399
|
+
(int64_t)offsets_view->data.as_int32[i + 1] - offsets_view->data.as_int32[i];
|
|
4400
|
+
view.data.as_uint8 = data_view + offsets_view->data.as_int32[i];
|
|
4401
|
+
break;
|
|
4402
|
+
case NANOARROW_TYPE_LARGE_STRING:
|
|
4403
|
+
case NANOARROW_TYPE_LARGE_BINARY:
|
|
4404
|
+
view.size_bytes =
|
|
4405
|
+
offsets_view->data.as_int64[i + 1] - offsets_view->data.as_int64[i];
|
|
4406
|
+
view.data.as_uint8 = data_view + offsets_view->data.as_int64[i];
|
|
4407
|
+
break;
|
|
4408
|
+
case NANOARROW_TYPE_FIXED_SIZE_BINARY:
|
|
4409
|
+
view.size_bytes = array_view->layout.element_size_bits[1] / 8;
|
|
4410
|
+
view.data.as_uint8 =
|
|
4411
|
+
array_view->buffer_views[1].data.as_uint8 + (i * view.size_bytes);
|
|
4412
|
+
break;
|
|
4413
|
+
case NANOARROW_TYPE_STRING_VIEW:
|
|
4414
|
+
case NANOARROW_TYPE_BINARY_VIEW:
|
|
4415
|
+
view = ArrowArrayViewGetBytesFromViewArrayUnsafe(array_view, i);
|
|
4416
|
+
break;
|
|
4417
|
+
default:
|
|
4418
|
+
view.data.data = NULL;
|
|
4419
|
+
view.size_bytes = 0;
|
|
4420
|
+
break;
|
|
4421
|
+
}
|
|
4422
|
+
|
|
4423
|
+
return view;
|
|
4424
|
+
}
|
|
4425
|
+
|
|
4426
|
+
static inline void ArrowArrayViewGetIntervalUnsafe(
|
|
4427
|
+
const struct ArrowArrayView* array_view, int64_t i, struct ArrowInterval* out) {
|
|
4428
|
+
const uint8_t* data_view = array_view->buffer_views[1].data.as_uint8;
|
|
4429
|
+
const int64_t offset = array_view->offset;
|
|
4430
|
+
const int64_t index = offset + i;
|
|
4431
|
+
switch (array_view->storage_type) {
|
|
4432
|
+
case NANOARROW_TYPE_INTERVAL_MONTHS: {
|
|
4433
|
+
const size_t size = sizeof(int32_t);
|
|
4434
|
+
memcpy(&out->months, data_view + index * size, sizeof(int32_t));
|
|
4435
|
+
break;
|
|
4436
|
+
}
|
|
4437
|
+
case NANOARROW_TYPE_INTERVAL_DAY_TIME: {
|
|
4438
|
+
const size_t size = sizeof(int32_t) + sizeof(int32_t);
|
|
4439
|
+
memcpy(&out->days, data_view + index * size, sizeof(int32_t));
|
|
4440
|
+
memcpy(&out->ms, data_view + index * size + 4, sizeof(int32_t));
|
|
4441
|
+
break;
|
|
4442
|
+
}
|
|
4443
|
+
case NANOARROW_TYPE_INTERVAL_MONTH_DAY_NANO: {
|
|
4444
|
+
const size_t size = sizeof(int32_t) + sizeof(int32_t) + sizeof(int64_t);
|
|
4445
|
+
memcpy(&out->months, data_view + index * size, sizeof(int32_t));
|
|
4446
|
+
memcpy(&out->days, data_view + index * size + 4, sizeof(int32_t));
|
|
4447
|
+
memcpy(&out->ns, data_view + index * size + 8, sizeof(int64_t));
|
|
4448
|
+
break;
|
|
4449
|
+
}
|
|
4450
|
+
default:
|
|
4451
|
+
break;
|
|
4452
|
+
}
|
|
4453
|
+
}
|
|
4454
|
+
|
|
4455
|
+
static inline void ArrowArrayViewGetDecimalUnsafe(const struct ArrowArrayView* array_view,
|
|
4456
|
+
int64_t i, struct ArrowDecimal* out) {
|
|
4457
|
+
i += array_view->offset;
|
|
4458
|
+
const uint8_t* data_view = array_view->buffer_views[1].data.as_uint8;
|
|
4459
|
+
switch (array_view->storage_type) {
|
|
4460
|
+
case NANOARROW_TYPE_DECIMAL32:
|
|
4461
|
+
ArrowDecimalSetBytes(out, data_view + (i * 4));
|
|
4462
|
+
break;
|
|
4463
|
+
case NANOARROW_TYPE_DECIMAL64:
|
|
4464
|
+
ArrowDecimalSetBytes(out, data_view + (i * 8));
|
|
4465
|
+
break;
|
|
4466
|
+
case NANOARROW_TYPE_DECIMAL128:
|
|
4467
|
+
ArrowDecimalSetBytes(out, data_view + (i * 16));
|
|
4468
|
+
break;
|
|
4469
|
+
case NANOARROW_TYPE_DECIMAL256:
|
|
4470
|
+
ArrowDecimalSetBytes(out, data_view + (i * 32));
|
|
4471
|
+
break;
|
|
4472
|
+
default:
|
|
4473
|
+
memset(out->words, 0, sizeof(out->words));
|
|
4474
|
+
break;
|
|
4475
|
+
}
|
|
4476
|
+
}
|
|
4477
|
+
|
|
4478
|
+
#ifdef __cplusplus
|
|
4479
|
+
}
|
|
4480
|
+
#endif
|
|
4481
|
+
|
|
4482
|
+
#endif
|