nanoarrow 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,4117 @@
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #include <errno.h>
19
+ #include <stdarg.h>
20
+ #include <stddef.h>
21
+ #include <stdio.h>
22
+ #include <stdlib.h>
23
+ #include <string.h>
24
+
25
+ #include "nanoarrow/nanoarrow.h"
26
+
27
+ const char* ArrowNanoarrowVersion(void) { return NANOARROW_VERSION; }
28
+
29
+ int ArrowNanoarrowVersionInt(void) { return NANOARROW_VERSION_INT; }
30
+
31
+ ArrowErrorCode ArrowErrorSet(struct ArrowError* error, const char* fmt, ...) {
32
+ if (error == NULL) {
33
+ return NANOARROW_OK;
34
+ }
35
+
36
+ memset(error->message, 0, sizeof(error->message));
37
+
38
+ va_list args;
39
+ va_start(args, fmt);
40
+ int chars_needed = vsnprintf(error->message, sizeof(error->message), fmt, args);
41
+ va_end(args);
42
+
43
+ if (chars_needed < 0) {
44
+ return EINVAL;
45
+ } else if (((size_t)chars_needed) >= sizeof(error->message)) {
46
+ return ERANGE;
47
+ } else {
48
+ return NANOARROW_OK;
49
+ }
50
+ }
51
+
52
+ void ArrowLayoutInit(struct ArrowLayout* layout, enum ArrowType storage_type) {
53
+ layout->buffer_type[0] = NANOARROW_BUFFER_TYPE_VALIDITY;
54
+ layout->buffer_data_type[0] = NANOARROW_TYPE_BOOL;
55
+ layout->buffer_type[1] = NANOARROW_BUFFER_TYPE_DATA;
56
+ layout->buffer_data_type[1] = storage_type;
57
+ layout->buffer_type[2] = NANOARROW_BUFFER_TYPE_NONE;
58
+ layout->buffer_data_type[2] = NANOARROW_TYPE_UNINITIALIZED;
59
+
60
+ layout->element_size_bits[0] = 1;
61
+ layout->element_size_bits[1] = 0;
62
+ layout->element_size_bits[2] = 0;
63
+
64
+ layout->child_size_elements = 0;
65
+
66
+ switch (storage_type) {
67
+ case NANOARROW_TYPE_UNINITIALIZED:
68
+ case NANOARROW_TYPE_NA:
69
+ case NANOARROW_TYPE_RUN_END_ENCODED:
70
+ layout->buffer_type[0] = NANOARROW_BUFFER_TYPE_NONE;
71
+ layout->buffer_data_type[0] = NANOARROW_TYPE_UNINITIALIZED;
72
+ layout->buffer_type[1] = NANOARROW_BUFFER_TYPE_NONE;
73
+ layout->buffer_data_type[1] = NANOARROW_TYPE_UNINITIALIZED;
74
+ layout->element_size_bits[0] = 0;
75
+ break;
76
+
77
+ case NANOARROW_TYPE_LIST:
78
+ case NANOARROW_TYPE_MAP:
79
+ layout->buffer_type[1] = NANOARROW_BUFFER_TYPE_DATA_OFFSET;
80
+ layout->buffer_data_type[1] = NANOARROW_TYPE_INT32;
81
+ layout->element_size_bits[1] = 32;
82
+ break;
83
+
84
+ case NANOARROW_TYPE_LARGE_LIST:
85
+ layout->buffer_type[1] = NANOARROW_BUFFER_TYPE_DATA_OFFSET;
86
+ layout->buffer_data_type[1] = NANOARROW_TYPE_INT64;
87
+ layout->element_size_bits[1] = 64;
88
+ break;
89
+
90
+ case NANOARROW_TYPE_STRUCT:
91
+ case NANOARROW_TYPE_FIXED_SIZE_LIST:
92
+ layout->buffer_type[1] = NANOARROW_BUFFER_TYPE_NONE;
93
+ layout->buffer_data_type[1] = NANOARROW_TYPE_UNINITIALIZED;
94
+ break;
95
+
96
+ case NANOARROW_TYPE_BOOL:
97
+ layout->element_size_bits[1] = 1;
98
+ break;
99
+
100
+ case NANOARROW_TYPE_UINT8:
101
+ case NANOARROW_TYPE_INT8:
102
+ layout->element_size_bits[1] = 8;
103
+ break;
104
+
105
+ case NANOARROW_TYPE_UINT16:
106
+ case NANOARROW_TYPE_INT16:
107
+ case NANOARROW_TYPE_HALF_FLOAT:
108
+ layout->element_size_bits[1] = 16;
109
+ break;
110
+
111
+ case NANOARROW_TYPE_UINT32:
112
+ case NANOARROW_TYPE_INT32:
113
+ case NANOARROW_TYPE_FLOAT:
114
+ case NANOARROW_TYPE_DECIMAL32:
115
+ layout->element_size_bits[1] = 32;
116
+ break;
117
+ case NANOARROW_TYPE_INTERVAL_MONTHS:
118
+ layout->buffer_data_type[1] = NANOARROW_TYPE_INT32;
119
+ layout->element_size_bits[1] = 32;
120
+ break;
121
+
122
+ case NANOARROW_TYPE_UINT64:
123
+ case NANOARROW_TYPE_INT64:
124
+ case NANOARROW_TYPE_DOUBLE:
125
+ case NANOARROW_TYPE_INTERVAL_DAY_TIME:
126
+ case NANOARROW_TYPE_DECIMAL64:
127
+ layout->element_size_bits[1] = 64;
128
+ break;
129
+
130
+ case NANOARROW_TYPE_DECIMAL128:
131
+ case NANOARROW_TYPE_INTERVAL_MONTH_DAY_NANO:
132
+ layout->element_size_bits[1] = 128;
133
+ break;
134
+
135
+ case NANOARROW_TYPE_DECIMAL256:
136
+ layout->element_size_bits[1] = 256;
137
+ break;
138
+
139
+ case NANOARROW_TYPE_FIXED_SIZE_BINARY:
140
+ layout->buffer_data_type[1] = NANOARROW_TYPE_BINARY;
141
+ break;
142
+
143
+ case NANOARROW_TYPE_DENSE_UNION:
144
+ layout->buffer_type[0] = NANOARROW_BUFFER_TYPE_TYPE_ID;
145
+ layout->buffer_data_type[0] = NANOARROW_TYPE_INT8;
146
+ layout->element_size_bits[0] = 8;
147
+ layout->buffer_type[1] = NANOARROW_BUFFER_TYPE_UNION_OFFSET;
148
+ layout->buffer_data_type[1] = NANOARROW_TYPE_INT32;
149
+ layout->element_size_bits[1] = 32;
150
+ break;
151
+
152
+ case NANOARROW_TYPE_SPARSE_UNION:
153
+ layout->buffer_type[0] = NANOARROW_BUFFER_TYPE_TYPE_ID;
154
+ layout->buffer_data_type[0] = NANOARROW_TYPE_INT8;
155
+ layout->element_size_bits[0] = 8;
156
+ layout->buffer_type[1] = NANOARROW_BUFFER_TYPE_NONE;
157
+ layout->buffer_data_type[1] = NANOARROW_TYPE_UNINITIALIZED;
158
+ break;
159
+
160
+ case NANOARROW_TYPE_STRING:
161
+ case NANOARROW_TYPE_BINARY:
162
+ layout->buffer_type[1] = NANOARROW_BUFFER_TYPE_DATA_OFFSET;
163
+ layout->buffer_data_type[1] = NANOARROW_TYPE_INT32;
164
+ layout->element_size_bits[1] = 32;
165
+ layout->buffer_type[2] = NANOARROW_BUFFER_TYPE_DATA;
166
+ layout->buffer_data_type[2] = storage_type;
167
+ break;
168
+
169
+ case NANOARROW_TYPE_LARGE_STRING:
170
+ layout->buffer_type[1] = NANOARROW_BUFFER_TYPE_DATA_OFFSET;
171
+ layout->buffer_data_type[1] = NANOARROW_TYPE_INT64;
172
+ layout->element_size_bits[1] = 64;
173
+ layout->buffer_type[2] = NANOARROW_BUFFER_TYPE_DATA;
174
+ layout->buffer_data_type[2] = NANOARROW_TYPE_STRING;
175
+ break;
176
+ case NANOARROW_TYPE_LARGE_BINARY:
177
+ layout->buffer_type[1] = NANOARROW_BUFFER_TYPE_DATA_OFFSET;
178
+ layout->buffer_data_type[1] = NANOARROW_TYPE_INT64;
179
+ layout->element_size_bits[1] = 64;
180
+ layout->buffer_type[2] = NANOARROW_BUFFER_TYPE_DATA;
181
+ layout->buffer_data_type[2] = NANOARROW_TYPE_BINARY;
182
+ break;
183
+
184
+ case NANOARROW_TYPE_BINARY_VIEW:
185
+ layout->buffer_type[1] = NANOARROW_BUFFER_TYPE_DATA;
186
+ layout->buffer_data_type[1] = NANOARROW_TYPE_BINARY_VIEW;
187
+ layout->element_size_bits[1] = 128;
188
+ break;
189
+ case NANOARROW_TYPE_STRING_VIEW:
190
+ layout->buffer_type[1] = NANOARROW_BUFFER_TYPE_DATA;
191
+ layout->buffer_data_type[1] = NANOARROW_TYPE_STRING_VIEW;
192
+ layout->element_size_bits[1] = 128;
193
+ break;
194
+
195
+ case NANOARROW_TYPE_LIST_VIEW:
196
+ layout->buffer_type[1] = NANOARROW_BUFFER_TYPE_VIEW_OFFSET;
197
+ layout->buffer_data_type[1] = NANOARROW_TYPE_INT32;
198
+ layout->element_size_bits[1] = 32;
199
+ layout->buffer_type[2] = NANOARROW_BUFFER_TYPE_SIZE;
200
+ layout->buffer_data_type[2] = NANOARROW_TYPE_INT32;
201
+ layout->element_size_bits[2] = 32;
202
+ break;
203
+ case NANOARROW_TYPE_LARGE_LIST_VIEW:
204
+ layout->buffer_type[1] = NANOARROW_BUFFER_TYPE_VIEW_OFFSET;
205
+ layout->buffer_data_type[1] = NANOARROW_TYPE_INT64;
206
+ layout->element_size_bits[1] = 64;
207
+ layout->buffer_type[2] = NANOARROW_BUFFER_TYPE_SIZE;
208
+ layout->buffer_data_type[2] = NANOARROW_TYPE_INT64;
209
+ layout->element_size_bits[2] = 64;
210
+ break;
211
+
212
+ default:
213
+ break;
214
+ }
215
+ }
216
+
217
+ void* ArrowMalloc(int64_t size) { return malloc(size); }
218
+
219
+ void* ArrowRealloc(void* ptr, int64_t size) { return realloc(ptr, size); }
220
+
221
+ void ArrowFree(void* ptr) { free(ptr); }
222
+
223
+ static uint8_t* ArrowBufferAllocatorMallocReallocate(
224
+ struct ArrowBufferAllocator* allocator, uint8_t* ptr, int64_t old_size,
225
+ int64_t new_size) {
226
+ NANOARROW_UNUSED(allocator);
227
+ NANOARROW_UNUSED(old_size);
228
+ return (uint8_t*)ArrowRealloc(ptr, new_size);
229
+ }
230
+
231
+ static void ArrowBufferAllocatorMallocFree(struct ArrowBufferAllocator* allocator,
232
+ uint8_t* ptr, int64_t size) {
233
+ NANOARROW_UNUSED(allocator);
234
+ NANOARROW_UNUSED(size);
235
+ if (ptr != NULL) {
236
+ ArrowFree(ptr);
237
+ }
238
+ }
239
+
240
+ static struct ArrowBufferAllocator ArrowBufferAllocatorMalloc = {
241
+ &ArrowBufferAllocatorMallocReallocate, &ArrowBufferAllocatorMallocFree, NULL};
242
+
243
+ struct ArrowBufferAllocator ArrowBufferAllocatorDefault(void) {
244
+ return ArrowBufferAllocatorMalloc;
245
+ }
246
+
247
+ static uint8_t* ArrowBufferDeallocatorReallocate(struct ArrowBufferAllocator* allocator,
248
+ uint8_t* ptr, int64_t old_size,
249
+ int64_t new_size) {
250
+ NANOARROW_UNUSED(new_size);
251
+
252
+ // Attempting to reallocate a buffer with a custom deallocator is
253
+ // a programming error. In debug mode, crash here.
254
+ #if defined(NANOARROW_DEBUG)
255
+ NANOARROW_PRINT_AND_DIE(ENOMEM,
256
+ "It is an error to reallocate a buffer whose allocator is "
257
+ "ArrowBufferDeallocator()");
258
+ #endif
259
+
260
+ // In release mode, ensure the the deallocator is called exactly
261
+ // once using the pointer it was given and return NULL, which
262
+ // will trigger the caller to return ENOMEM.
263
+ allocator->free(allocator, ptr, old_size);
264
+ *allocator = ArrowBufferAllocatorDefault();
265
+ return NULL;
266
+ }
267
+
268
+ struct ArrowBufferAllocator ArrowBufferDeallocator(
269
+ void (*custom_free)(struct ArrowBufferAllocator* allocator, uint8_t* ptr,
270
+ int64_t size),
271
+ void* private_data) {
272
+ struct ArrowBufferAllocator allocator;
273
+ allocator.reallocate = &ArrowBufferDeallocatorReallocate;
274
+ allocator.free = custom_free;
275
+ allocator.private_data = private_data;
276
+ return allocator;
277
+ }
278
+
279
+ static const int kInt32DecimalDigits = 9;
280
+
281
+ static const uint64_t kUInt32PowersOfTen[] = {
282
+ 1ULL, 10ULL, 100ULL, 1000ULL, 10000ULL,
283
+ 100000ULL, 1000000ULL, 10000000ULL, 100000000ULL, 1000000000ULL};
284
+
285
+ // Adapted from Arrow C++ to use 32-bit words for better C portability
286
+ // https://github.com/apache/arrow/blob/cd3321b28b0c9703e5d7105d6146c1270bbadd7f/cpp/src/arrow/util/decimal.cc#L524-L544
287
+ static void ShiftAndAdd(struct ArrowStringView value, uint32_t* out, int64_t out_size) {
288
+ // We use strtoll for parsing, which needs input that is null-terminated
289
+ char chunk_string[16];
290
+
291
+ for (int64_t posn = 0; posn < value.size_bytes;) {
292
+ int64_t remaining = value.size_bytes - posn;
293
+
294
+ int64_t group_size;
295
+ if (remaining > kInt32DecimalDigits) {
296
+ group_size = kInt32DecimalDigits;
297
+ } else {
298
+ group_size = remaining;
299
+ }
300
+
301
+ const uint64_t multiple = kUInt32PowersOfTen[group_size];
302
+
303
+ memcpy(chunk_string, value.data + posn, group_size);
304
+ chunk_string[group_size] = '\0';
305
+ uint32_t chunk = (uint32_t)strtoll(chunk_string, NULL, 10);
306
+
307
+ for (int64_t i = 0; i < out_size; i++) {
308
+ uint64_t tmp = out[i];
309
+ tmp *= multiple;
310
+ tmp += chunk;
311
+ out[i] = (uint32_t)(tmp & 0xFFFFFFFFULL);
312
+ chunk = (uint32_t)(tmp >> 32);
313
+ }
314
+ posn += group_size;
315
+ }
316
+ }
317
+
318
+ ArrowErrorCode ArrowDecimalSetDigits(struct ArrowDecimal* decimal,
319
+ struct ArrowStringView value) {
320
+ // Check for sign
321
+ int is_negative = value.data[0] == '-';
322
+ int has_sign = is_negative || value.data[0] == '+';
323
+ value.data += has_sign;
324
+ value.size_bytes -= has_sign;
325
+
326
+ // Check all characters are digits that are not the negative sign
327
+ for (int64_t i = 0; i < value.size_bytes; i++) {
328
+ char c = value.data[i];
329
+ if (c < '0' || c > '9') {
330
+ return EINVAL;
331
+ }
332
+ }
333
+
334
+ // Skip over leading 0s
335
+ int64_t n_leading_zeroes = 0;
336
+ for (int64_t i = 0; i < value.size_bytes; i++) {
337
+ if (value.data[i] == '0') {
338
+ n_leading_zeroes++;
339
+ } else {
340
+ break;
341
+ }
342
+ }
343
+
344
+ value.data += n_leading_zeroes;
345
+ value.size_bytes -= n_leading_zeroes;
346
+
347
+ // Use 32-bit words for portability
348
+ uint32_t words32[8];
349
+ memset(words32, 0, sizeof(words32));
350
+ int n_words32 = decimal->n_words > 0 ? decimal->n_words * 2 : 1;
351
+ NANOARROW_DCHECK(n_words32 <= 8);
352
+ memset(words32, 0, sizeof(words32));
353
+
354
+ ShiftAndAdd(value, words32, n_words32);
355
+
356
+ if (_ArrowIsLittleEndian() || n_words32 == 1) {
357
+ memcpy(decimal->words, words32, sizeof(uint32_t) * n_words32);
358
+ } else {
359
+ uint64_t lo;
360
+ uint64_t hi;
361
+
362
+ for (int i = 0; i < decimal->n_words; i++) {
363
+ lo = (uint64_t)words32[i * 2];
364
+ hi = (uint64_t)words32[i * 2 + 1] << 32;
365
+ decimal->words[decimal->n_words - i - 1] = lo | hi;
366
+ }
367
+ }
368
+
369
+ if (is_negative) {
370
+ ArrowDecimalNegate(decimal);
371
+ }
372
+
373
+ return NANOARROW_OK;
374
+ }
375
+
376
+ // Adapted from Arrow C++ for C
377
+ // https://github.com/apache/arrow/blob/cd3321b28b0c9703e5d7105d6146c1270bbadd7f/cpp/src/arrow/util/decimal.cc#L365
378
+ ArrowErrorCode ArrowDecimalAppendDigitsToBuffer(const struct ArrowDecimal* decimal,
379
+ struct ArrowBuffer* buffer) {
380
+ NANOARROW_DCHECK(decimal->n_words == 0 || decimal->n_words == 1 ||
381
+ decimal->n_words == 2 || decimal->n_words == 4);
382
+
383
+ // For the 32-bit case, just use snprintf()
384
+ if (decimal->n_words == 0) {
385
+ int32_t value;
386
+ memcpy(&value, decimal->words, sizeof(int32_t));
387
+ NANOARROW_RETURN_NOT_OK(ArrowBufferReserve(buffer, 16));
388
+ int n_chars = snprintf((char*)buffer->data + buffer->size_bytes,
389
+ (buffer->capacity_bytes - buffer->size_bytes), "%d", value);
390
+ if (n_chars <= 0) {
391
+ return EINVAL;
392
+ }
393
+
394
+ buffer->size_bytes += n_chars;
395
+ return NANOARROW_OK;
396
+ }
397
+
398
+ int is_negative = ArrowDecimalSign(decimal) < 0;
399
+
400
+ uint64_t words_little_endian[4];
401
+ if (decimal->n_words == 0) {
402
+ words_little_endian[0] = 0;
403
+ memcpy(words_little_endian, decimal->words, sizeof(uint32_t));
404
+ } else if (decimal->low_word_index == 0) {
405
+ memcpy(words_little_endian, decimal->words, decimal->n_words * sizeof(uint64_t));
406
+ } else {
407
+ for (int i = 0; i < decimal->n_words; i++) {
408
+ words_little_endian[i] = decimal->words[decimal->n_words - i - 1];
409
+ }
410
+ }
411
+
412
+ // We've already made a copy, so negate that if needed
413
+ if (is_negative) {
414
+ if (decimal->n_words == 0) {
415
+ uint32_t elem = (uint32_t)words_little_endian[0];
416
+ elem = ~elem + 1;
417
+ words_little_endian[0] = (int32_t)elem;
418
+ } else {
419
+ uint64_t carry = 1;
420
+ for (int i = 0; i < decimal->n_words; i++) {
421
+ uint64_t elem = words_little_endian[i];
422
+ elem = ~elem + carry;
423
+ carry &= (elem == 0);
424
+ words_little_endian[i] = elem;
425
+ }
426
+ }
427
+ }
428
+
429
+ // Find the most significant word that is non-zero
430
+ int most_significant_elem_idx = -1;
431
+ if (decimal->n_words == 0) {
432
+ if (words_little_endian[0] != 0) {
433
+ most_significant_elem_idx = 0;
434
+ }
435
+ } else {
436
+ for (int i = decimal->n_words - 1; i >= 0; i--) {
437
+ if (words_little_endian[i] != 0) {
438
+ most_significant_elem_idx = i;
439
+ break;
440
+ }
441
+ }
442
+ }
443
+
444
+ // If they are all zero, the output is just '0'
445
+ if (most_significant_elem_idx == -1) {
446
+ NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt8(buffer, '0'));
447
+ return NANOARROW_OK;
448
+ }
449
+
450
+ // Define segments such that each segment represents 9 digits with the
451
+ // least significant group of 9 digits first. For example, if the input represents
452
+ // 9876543210123456789, then segments will be [123456789, 876543210, 9].
453
+ // We handle at most a signed 256 bit integer, whose maximum value occupies 77
454
+ // characters. Thus, we need at most 9 segments.
455
+ const uint32_t k1e9 = 1000000000U;
456
+ int num_segments = 0;
457
+ uint32_t segments[9];
458
+ memset(segments, 0, sizeof(segments));
459
+ uint64_t* most_significant_elem = words_little_endian + most_significant_elem_idx;
460
+
461
+ do {
462
+ // Compute remainder = words_little_endian % 1e9 and words_little_endian =
463
+ // words_little_endian / 1e9.
464
+ uint32_t remainder = 0;
465
+ uint64_t* elem = most_significant_elem;
466
+
467
+ do {
468
+ // Compute dividend = (remainder << 32) | *elem (a virtual 96-bit integer);
469
+ // *elem = dividend / 1e9;
470
+ // remainder = dividend % 1e9.
471
+ uint32_t hi = (uint32_t)(*elem >> 32);
472
+ uint32_t lo = (uint32_t)(*elem & 0xFFFFFFFFULL);
473
+ uint64_t dividend_hi = ((uint64_t)(remainder) << 32) | hi;
474
+ uint64_t quotient_hi = dividend_hi / k1e9;
475
+ remainder = (uint32_t)(dividend_hi % k1e9);
476
+ uint64_t dividend_lo = ((uint64_t)(remainder) << 32) | lo;
477
+ uint64_t quotient_lo = dividend_lo / k1e9;
478
+ remainder = (uint32_t)(dividend_lo % k1e9);
479
+
480
+ *elem = (quotient_hi << 32) | quotient_lo;
481
+ } while (elem-- != words_little_endian);
482
+
483
+ segments[num_segments++] = remainder;
484
+ } while (*most_significant_elem != 0 || most_significant_elem-- != words_little_endian);
485
+
486
+ // We know our output has no more than 9 digits per segment, plus a negative sign,
487
+ // plus any further digits between our output of 9 digits plus enough
488
+ // extra characters to ensure that snprintf() with n = 21 (maximum length of %lu
489
+ // including a the null terminator) is bounded properly.
490
+ NANOARROW_RETURN_NOT_OK(ArrowBufferReserve(buffer, num_segments * 9 + 1 + 21 - 9));
491
+ if (is_negative) {
492
+ buffer->data[buffer->size_bytes++] = '-';
493
+ }
494
+
495
+ // The most significant segment should have no leading zeroes
496
+ int n_chars = snprintf((char*)buffer->data + buffer->size_bytes, 21, "%lu",
497
+ (unsigned long)segments[num_segments - 1]);
498
+
499
+ // Ensure that an encoding error from snprintf() does not result
500
+ // in an out-of-bounds access.
501
+ if (n_chars < 0) {
502
+ return ERANGE;
503
+ }
504
+
505
+ buffer->size_bytes += n_chars;
506
+
507
+ // Subsequent output needs to be left-padded with zeroes such that each segment
508
+ // takes up exactly 9 digits.
509
+ for (int i = num_segments - 2; i >= 0; i--) {
510
+ int n_chars = snprintf((char*)buffer->data + buffer->size_bytes, 21, "%09lu",
511
+ (unsigned long)segments[i]);
512
+ buffer->size_bytes += n_chars;
513
+ NANOARROW_DCHECK(buffer->size_bytes <= buffer->capacity_bytes);
514
+ }
515
+
516
+ return NANOARROW_OK;
517
+ }
518
+
519
+ ArrowErrorCode ArrowDecimalAppendStringToBuffer(const struct ArrowDecimal* decimal,
520
+ struct ArrowBuffer* buffer) {
521
+ int64_t buffer_size = buffer->size_bytes;
522
+ NANOARROW_RETURN_NOT_OK(ArrowDecimalAppendDigitsToBuffer(decimal, buffer));
523
+ int64_t digits_size = buffer->size_bytes - buffer_size;
524
+
525
+ if (decimal->scale <= 0) {
526
+ // e.g., digits are -12345 and scale is -2 -> -1234500
527
+ // Just add zeros to the end
528
+ for (int i = decimal->scale; i < 0; i++) {
529
+ NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt8(buffer, '0'));
530
+ }
531
+ return NANOARROW_OK;
532
+ }
533
+
534
+ int is_negative = buffer->data[0] == '-';
535
+ int64_t num_digits = digits_size - is_negative;
536
+ if (num_digits <= decimal->scale) {
537
+ // e.g., digits are -12345 and scale is 6 -> -0.012345
538
+ // Insert "0.<some zeros>" between the (maybe) negative sign and the digits
539
+ int64_t num_zeros_after_decimal = decimal->scale - num_digits;
540
+ NANOARROW_RETURN_NOT_OK(
541
+ ArrowBufferResize(buffer, buffer->size_bytes + num_zeros_after_decimal + 2, 0));
542
+
543
+ uint8_t* digits_start = buffer->data + is_negative;
544
+ memmove(digits_start + num_zeros_after_decimal + 2, digits_start, num_digits);
545
+ *digits_start++ = '0';
546
+ *digits_start++ = '.';
547
+ for (int i = 0; i < num_zeros_after_decimal; i++) {
548
+ *digits_start++ = '0';
549
+ }
550
+
551
+ } else {
552
+ // e.g., digits are -12345 and scale is 4 -> -1.2345
553
+ // Insert a decimal point before scale digits of output
554
+ NANOARROW_RETURN_NOT_OK(ArrowBufferResize(buffer, buffer->size_bytes + 1, 0));
555
+ uint8_t* decimal_point_to_be = buffer->data + buffer->size_bytes - 1 - decimal->scale;
556
+ memmove(decimal_point_to_be + 1, decimal_point_to_be, decimal->scale);
557
+ *decimal_point_to_be = '.';
558
+ }
559
+
560
+ return NANOARROW_OK;
561
+ }
562
+ // Licensed to the Apache Software Foundation (ASF) under one
563
+ // or more contributor license agreements. See the NOTICE file
564
+ // distributed with this work for additional information
565
+ // regarding copyright ownership. The ASF licenses this file
566
+ // to you under the Apache License, Version 2.0 (the
567
+ // "License"); you may not use this file except in compliance
568
+ // with the License. You may obtain a copy of the License at
569
+ //
570
+ // http://www.apache.org/licenses/LICENSE-2.0
571
+ //
572
+ // Unless required by applicable law or agreed to in writing,
573
+ // software distributed under the License is distributed on an
574
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
575
+ // KIND, either express or implied. See the License for the
576
+ // specific language governing permissions and limitations
577
+ // under the License.
578
+
579
+ #include <errno.h>
580
+ #include <inttypes.h>
581
+ #include <stdio.h>
582
+ #include <stdlib.h>
583
+ #include <string.h>
584
+
585
+ #include "nanoarrow/nanoarrow.h"
586
+
587
+ static void ArrowSchemaReleaseInternal(struct ArrowSchema* schema) {
588
+ if (schema->format != NULL) ArrowFree((void*)schema->format);
589
+ if (schema->name != NULL) ArrowFree((void*)schema->name);
590
+ if (schema->metadata != NULL) ArrowFree((void*)schema->metadata);
591
+
592
+ // This object owns the memory for all the children, but those
593
+ // children may have been generated elsewhere and might have
594
+ // their own release() callback.
595
+ if (schema->children != NULL) {
596
+ for (int64_t i = 0; i < schema->n_children; i++) {
597
+ if (schema->children[i] != NULL) {
598
+ if (schema->children[i]->release != NULL) {
599
+ ArrowSchemaRelease(schema->children[i]);
600
+ }
601
+
602
+ ArrowFree(schema->children[i]);
603
+ }
604
+ }
605
+
606
+ ArrowFree(schema->children);
607
+ }
608
+
609
+ // This object owns the memory for the dictionary but it
610
+ // may have been generated somewhere else and have its own
611
+ // release() callback.
612
+ if (schema->dictionary != NULL) {
613
+ if (schema->dictionary->release != NULL) {
614
+ ArrowSchemaRelease(schema->dictionary);
615
+ }
616
+
617
+ ArrowFree(schema->dictionary);
618
+ }
619
+
620
+ // private data not currently used
621
+ if (schema->private_data != NULL) {
622
+ ArrowFree(schema->private_data);
623
+ }
624
+
625
+ schema->release = NULL;
626
+ }
627
+
628
+ static const char* ArrowSchemaFormatTemplate(enum ArrowType type) {
629
+ switch (type) {
630
+ case NANOARROW_TYPE_UNINITIALIZED:
631
+ return NULL;
632
+ case NANOARROW_TYPE_NA:
633
+ return "n";
634
+ case NANOARROW_TYPE_BOOL:
635
+ return "b";
636
+
637
+ case NANOARROW_TYPE_UINT8:
638
+ return "C";
639
+ case NANOARROW_TYPE_INT8:
640
+ return "c";
641
+ case NANOARROW_TYPE_UINT16:
642
+ return "S";
643
+ case NANOARROW_TYPE_INT16:
644
+ return "s";
645
+ case NANOARROW_TYPE_UINT32:
646
+ return "I";
647
+ case NANOARROW_TYPE_INT32:
648
+ return "i";
649
+ case NANOARROW_TYPE_UINT64:
650
+ return "L";
651
+ case NANOARROW_TYPE_INT64:
652
+ return "l";
653
+
654
+ case NANOARROW_TYPE_HALF_FLOAT:
655
+ return "e";
656
+ case NANOARROW_TYPE_FLOAT:
657
+ return "f";
658
+ case NANOARROW_TYPE_DOUBLE:
659
+ return "g";
660
+
661
+ case NANOARROW_TYPE_STRING:
662
+ return "u";
663
+ case NANOARROW_TYPE_LARGE_STRING:
664
+ return "U";
665
+ case NANOARROW_TYPE_STRING_VIEW:
666
+ return "vu";
667
+ case NANOARROW_TYPE_BINARY:
668
+ return "z";
669
+ case NANOARROW_TYPE_BINARY_VIEW:
670
+ return "vz";
671
+ case NANOARROW_TYPE_LARGE_BINARY:
672
+ return "Z";
673
+
674
+ case NANOARROW_TYPE_DATE32:
675
+ return "tdD";
676
+ case NANOARROW_TYPE_DATE64:
677
+ return "tdm";
678
+ case NANOARROW_TYPE_INTERVAL_MONTHS:
679
+ return "tiM";
680
+ case NANOARROW_TYPE_INTERVAL_DAY_TIME:
681
+ return "tiD";
682
+ case NANOARROW_TYPE_INTERVAL_MONTH_DAY_NANO:
683
+ return "tin";
684
+
685
+ case NANOARROW_TYPE_LIST:
686
+ return "+l";
687
+ case NANOARROW_TYPE_LARGE_LIST:
688
+ return "+L";
689
+ case NANOARROW_TYPE_LIST_VIEW:
690
+ return "+vl";
691
+ case NANOARROW_TYPE_LARGE_LIST_VIEW:
692
+ return "+vL";
693
+ case NANOARROW_TYPE_STRUCT:
694
+ return "+s";
695
+ case NANOARROW_TYPE_MAP:
696
+ return "+m";
697
+ case NANOARROW_TYPE_RUN_END_ENCODED:
698
+ return "+r";
699
+
700
+ default:
701
+ return NULL;
702
+ }
703
+ }
704
+
705
+ static int ArrowSchemaInitChildrenIfNeeded(struct ArrowSchema* schema,
706
+ enum ArrowType type) {
707
+ switch (type) {
708
+ case NANOARROW_TYPE_LIST:
709
+ case NANOARROW_TYPE_LARGE_LIST:
710
+ case NANOARROW_TYPE_FIXED_SIZE_LIST:
711
+ case NANOARROW_TYPE_LIST_VIEW:
712
+ case NANOARROW_TYPE_LARGE_LIST_VIEW:
713
+ NANOARROW_RETURN_NOT_OK(ArrowSchemaAllocateChildren(schema, 1));
714
+ ArrowSchemaInit(schema->children[0]);
715
+ NANOARROW_RETURN_NOT_OK(ArrowSchemaSetName(schema->children[0], "item"));
716
+ break;
717
+ case NANOARROW_TYPE_MAP:
718
+ NANOARROW_RETURN_NOT_OK(ArrowSchemaAllocateChildren(schema, 1));
719
+ NANOARROW_RETURN_NOT_OK(
720
+ ArrowSchemaInitFromType(schema->children[0], NANOARROW_TYPE_STRUCT));
721
+ NANOARROW_RETURN_NOT_OK(ArrowSchemaSetName(schema->children[0], "entries"));
722
+ schema->children[0]->flags &= ~ARROW_FLAG_NULLABLE;
723
+ NANOARROW_RETURN_NOT_OK(ArrowSchemaAllocateChildren(schema->children[0], 2));
724
+ ArrowSchemaInit(schema->children[0]->children[0]);
725
+ ArrowSchemaInit(schema->children[0]->children[1]);
726
+ NANOARROW_RETURN_NOT_OK(
727
+ ArrowSchemaSetName(schema->children[0]->children[0], "key"));
728
+ schema->children[0]->children[0]->flags &= ~ARROW_FLAG_NULLABLE;
729
+ NANOARROW_RETURN_NOT_OK(
730
+ ArrowSchemaSetName(schema->children[0]->children[1], "value"));
731
+ break;
732
+ case NANOARROW_TYPE_RUN_END_ENCODED:
733
+ NANOARROW_RETURN_NOT_OK(ArrowSchemaAllocateChildren(schema, 2));
734
+ ArrowSchemaInit(schema->children[0]);
735
+ NANOARROW_RETURN_NOT_OK(ArrowSchemaSetName(schema->children[0], "run_ends"));
736
+ schema->children[0]->flags &= ~ARROW_FLAG_NULLABLE;
737
+ ArrowSchemaInit(schema->children[1]);
738
+ NANOARROW_RETURN_NOT_OK(ArrowSchemaSetName(schema->children[1], "values"));
739
+ default:
740
+ break;
741
+ }
742
+
743
+ return NANOARROW_OK;
744
+ }
745
+
746
+ void ArrowSchemaInit(struct ArrowSchema* schema) {
747
+ schema->format = NULL;
748
+ schema->name = NULL;
749
+ schema->metadata = NULL;
750
+ schema->flags = ARROW_FLAG_NULLABLE;
751
+ schema->n_children = 0;
752
+ schema->children = NULL;
753
+ schema->dictionary = NULL;
754
+ schema->private_data = NULL;
755
+ schema->release = &ArrowSchemaReleaseInternal;
756
+ }
757
+
758
+ ArrowErrorCode ArrowSchemaSetType(struct ArrowSchema* schema, enum ArrowType type) {
759
+ // We don't allocate the dictionary because it has to be nullptr
760
+ // for non-dictionary-encoded arrays.
761
+
762
+ // Set the format to a valid format string for type
763
+ const char* template_format = ArrowSchemaFormatTemplate(type);
764
+
765
+ // If type isn't recognized and not explicitly unset
766
+ if (template_format == NULL && type != NANOARROW_TYPE_UNINITIALIZED) {
767
+ return EINVAL;
768
+ }
769
+
770
+ NANOARROW_RETURN_NOT_OK(ArrowSchemaSetFormat(schema, template_format));
771
+
772
+ // For types with an umabiguous child structure, allocate children
773
+ return ArrowSchemaInitChildrenIfNeeded(schema, type);
774
+ }
775
+
776
+ ArrowErrorCode ArrowSchemaSetTypeStruct(struct ArrowSchema* schema, int64_t n_children) {
777
+ NANOARROW_RETURN_NOT_OK(ArrowSchemaSetType(schema, NANOARROW_TYPE_STRUCT));
778
+ NANOARROW_RETURN_NOT_OK(ArrowSchemaAllocateChildren(schema, n_children));
779
+ for (int64_t i = 0; i < n_children; i++) {
780
+ ArrowSchemaInit(schema->children[i]);
781
+ }
782
+
783
+ return NANOARROW_OK;
784
+ }
785
+
786
+ ArrowErrorCode ArrowSchemaInitFromType(struct ArrowSchema* schema, enum ArrowType type) {
787
+ ArrowSchemaInit(schema);
788
+
789
+ int result = ArrowSchemaSetType(schema, type);
790
+ if (result != NANOARROW_OK) {
791
+ ArrowSchemaRelease(schema);
792
+ return result;
793
+ }
794
+
795
+ return NANOARROW_OK;
796
+ }
797
+
798
+ ArrowErrorCode ArrowSchemaSetTypeFixedSize(struct ArrowSchema* schema,
799
+ enum ArrowType type, int32_t fixed_size) {
800
+ if (fixed_size <= 0) {
801
+ return EINVAL;
802
+ }
803
+
804
+ char buffer[64];
805
+ int n_chars;
806
+ switch (type) {
807
+ case NANOARROW_TYPE_FIXED_SIZE_BINARY:
808
+ n_chars = snprintf(buffer, sizeof(buffer), "w:%" PRId32, fixed_size);
809
+ break;
810
+ case NANOARROW_TYPE_FIXED_SIZE_LIST:
811
+ n_chars = snprintf(buffer, sizeof(buffer), "+w:%" PRId32, fixed_size);
812
+ break;
813
+ default:
814
+ return EINVAL;
815
+ }
816
+
817
+ if (((size_t)n_chars) >= sizeof(buffer) || n_chars < 0) {
818
+ return ERANGE;
819
+ }
820
+
821
+ buffer[n_chars] = '\0';
822
+ NANOARROW_RETURN_NOT_OK(ArrowSchemaSetFormat(schema, buffer));
823
+
824
+ if (type == NANOARROW_TYPE_FIXED_SIZE_LIST) {
825
+ NANOARROW_RETURN_NOT_OK(ArrowSchemaInitChildrenIfNeeded(schema, type));
826
+ }
827
+
828
+ return NANOARROW_OK;
829
+ }
830
+
831
+ ArrowErrorCode ArrowSchemaSetTypeDecimal(struct ArrowSchema* schema, enum ArrowType type,
832
+ int32_t decimal_precision,
833
+ int32_t decimal_scale) {
834
+ if (decimal_precision <= 0) {
835
+ return EINVAL;
836
+ }
837
+
838
+ char buffer[64];
839
+ int n_chars;
840
+ switch (type) {
841
+ case NANOARROW_TYPE_DECIMAL32:
842
+ if (decimal_precision > 9) {
843
+ return EINVAL;
844
+ }
845
+
846
+ n_chars = snprintf(buffer, sizeof(buffer), "d:%d,%d,32", decimal_precision,
847
+ decimal_scale);
848
+ break;
849
+ case NANOARROW_TYPE_DECIMAL64:
850
+ if (decimal_precision > 18) {
851
+ return EINVAL;
852
+ }
853
+
854
+ n_chars = snprintf(buffer, sizeof(buffer), "d:%d,%d,64", decimal_precision,
855
+ decimal_scale);
856
+ break;
857
+ case NANOARROW_TYPE_DECIMAL128:
858
+ if (decimal_precision > 38) {
859
+ return EINVAL;
860
+ }
861
+
862
+ n_chars =
863
+ snprintf(buffer, sizeof(buffer), "d:%d,%d", decimal_precision, decimal_scale);
864
+ break;
865
+ case NANOARROW_TYPE_DECIMAL256:
866
+ if (decimal_precision > 76) {
867
+ return EINVAL;
868
+ }
869
+
870
+ n_chars = snprintf(buffer, sizeof(buffer), "d:%d,%d,256", decimal_precision,
871
+ decimal_scale);
872
+ break;
873
+ default:
874
+ return EINVAL;
875
+ }
876
+
877
+ if (((size_t)n_chars) >= sizeof(buffer) || n_chars < 0) {
878
+ return ERANGE;
879
+ }
880
+
881
+ buffer[n_chars] = '\0';
882
+ return ArrowSchemaSetFormat(schema, buffer);
883
+ }
884
+
885
+ ArrowErrorCode ArrowSchemaSetTypeRunEndEncoded(struct ArrowSchema* schema,
886
+ enum ArrowType run_end_type) {
887
+ switch (run_end_type) {
888
+ case NANOARROW_TYPE_INT16:
889
+ case NANOARROW_TYPE_INT32:
890
+ case NANOARROW_TYPE_INT64:
891
+ break;
892
+ default:
893
+ return EINVAL;
894
+ }
895
+
896
+ NANOARROW_RETURN_NOT_OK(ArrowSchemaSetFormat(
897
+ schema, ArrowSchemaFormatTemplate(NANOARROW_TYPE_RUN_END_ENCODED)));
898
+ NANOARROW_RETURN_NOT_OK(
899
+ ArrowSchemaInitChildrenIfNeeded(schema, NANOARROW_TYPE_RUN_END_ENCODED));
900
+ NANOARROW_RETURN_NOT_OK(ArrowSchemaSetType(schema->children[0], run_end_type));
901
+ NANOARROW_RETURN_NOT_OK(
902
+ ArrowSchemaSetType(schema->children[1], NANOARROW_TYPE_UNINITIALIZED));
903
+
904
+ return NANOARROW_OK;
905
+ }
906
+
907
+ static const char* ArrowTimeUnitFormatString(enum ArrowTimeUnit time_unit) {
908
+ switch (time_unit) {
909
+ case NANOARROW_TIME_UNIT_SECOND:
910
+ return "s";
911
+ case NANOARROW_TIME_UNIT_MILLI:
912
+ return "m";
913
+ case NANOARROW_TIME_UNIT_MICRO:
914
+ return "u";
915
+ case NANOARROW_TIME_UNIT_NANO:
916
+ return "n";
917
+ default:
918
+ return NULL;
919
+ }
920
+ }
921
+
922
+ ArrowErrorCode ArrowSchemaSetTypeDateTime(struct ArrowSchema* schema, enum ArrowType type,
923
+ enum ArrowTimeUnit time_unit,
924
+ const char* timezone) {
925
+ const char* time_unit_str = ArrowTimeUnitFormatString(time_unit);
926
+ if (time_unit_str == NULL) {
927
+ return EINVAL;
928
+ }
929
+
930
+ char buffer[128];
931
+ int n_chars;
932
+ switch (type) {
933
+ case NANOARROW_TYPE_TIME32:
934
+ if (timezone != NULL) {
935
+ return EINVAL;
936
+ }
937
+
938
+ switch (time_unit) {
939
+ case NANOARROW_TIME_UNIT_MICRO:
940
+ case NANOARROW_TIME_UNIT_NANO:
941
+ return EINVAL;
942
+ default:
943
+ break;
944
+ }
945
+
946
+ n_chars = snprintf(buffer, sizeof(buffer), "tt%s", time_unit_str);
947
+ break;
948
+ case NANOARROW_TYPE_TIME64:
949
+ if (timezone != NULL) {
950
+ return EINVAL;
951
+ }
952
+
953
+ switch (time_unit) {
954
+ case NANOARROW_TIME_UNIT_SECOND:
955
+ case NANOARROW_TIME_UNIT_MILLI:
956
+ return EINVAL;
957
+ default:
958
+ break;
959
+ }
960
+
961
+ n_chars = snprintf(buffer, sizeof(buffer), "tt%s", time_unit_str);
962
+ break;
963
+ case NANOARROW_TYPE_TIMESTAMP:
964
+ if (timezone == NULL) {
965
+ timezone = "";
966
+ }
967
+ n_chars = snprintf(buffer, sizeof(buffer), "ts%s:%s", time_unit_str, timezone);
968
+ break;
969
+ case NANOARROW_TYPE_DURATION:
970
+ if (timezone != NULL) {
971
+ return EINVAL;
972
+ }
973
+ n_chars = snprintf(buffer, sizeof(buffer), "tD%s", time_unit_str);
974
+ break;
975
+ default:
976
+ return EINVAL;
977
+ }
978
+
979
+ if (((size_t)n_chars) >= sizeof(buffer) || n_chars < 0) {
980
+ return ERANGE;
981
+ }
982
+
983
+ buffer[n_chars] = '\0';
984
+
985
+ return ArrowSchemaSetFormat(schema, buffer);
986
+ }
987
+
988
+ ArrowErrorCode ArrowSchemaSetTypeUnion(struct ArrowSchema* schema, enum ArrowType type,
989
+ int64_t n_children) {
990
+ if (n_children < 0 || n_children > 127) {
991
+ return EINVAL;
992
+ }
993
+
994
+ // Max valid size would be +ud:0,1,...126 = 401 characters + null terminator
995
+ char format_out[512];
996
+ int64_t format_out_size = 512;
997
+ memset(format_out, 0, format_out_size);
998
+ int n_chars;
999
+ char* format_cursor = format_out;
1000
+
1001
+ switch (type) {
1002
+ case NANOARROW_TYPE_SPARSE_UNION:
1003
+ n_chars = snprintf(format_cursor, format_out_size, "+us:");
1004
+ format_cursor += n_chars;
1005
+ format_out_size -= n_chars;
1006
+ break;
1007
+ case NANOARROW_TYPE_DENSE_UNION:
1008
+ n_chars = snprintf(format_cursor, format_out_size, "+ud:");
1009
+ format_cursor += n_chars;
1010
+ format_out_size -= n_chars;
1011
+ break;
1012
+ default:
1013
+ return EINVAL;
1014
+ }
1015
+
1016
+ // Ensure that an encoding error from snprintf() does not result
1017
+ // in an out-of-bounds access.
1018
+ if (n_chars < 0) {
1019
+ return ERANGE;
1020
+ }
1021
+
1022
+ if (n_children > 0) {
1023
+ n_chars = snprintf(format_cursor, format_out_size, "0");
1024
+ format_cursor += n_chars;
1025
+ format_out_size -= n_chars;
1026
+
1027
+ for (int64_t i = 1; i < n_children; i++) {
1028
+ n_chars = snprintf(format_cursor, format_out_size, ",%" PRId64, i);
1029
+ format_cursor += n_chars;
1030
+ format_out_size -= n_chars;
1031
+ }
1032
+ }
1033
+
1034
+ // Ensure that an encoding error from snprintf() does not result
1035
+ // in an out-of-bounds access.
1036
+ if (n_chars < 0) {
1037
+ return ERANGE;
1038
+ }
1039
+
1040
+ NANOARROW_RETURN_NOT_OK(ArrowSchemaSetFormat(schema, format_out));
1041
+
1042
+ NANOARROW_RETURN_NOT_OK(ArrowSchemaAllocateChildren(schema, n_children));
1043
+ for (int64_t i = 0; i < n_children; i++) {
1044
+ ArrowSchemaInit(schema->children[i]);
1045
+ }
1046
+
1047
+ return NANOARROW_OK;
1048
+ }
1049
+
1050
+ ArrowErrorCode ArrowSchemaSetFormat(struct ArrowSchema* schema, const char* format) {
1051
+ if (schema->format != NULL) {
1052
+ ArrowFree((void*)schema->format);
1053
+ }
1054
+
1055
+ if (format != NULL) {
1056
+ size_t format_size = strlen(format) + 1;
1057
+ schema->format = (const char*)ArrowMalloc(format_size);
1058
+ if (schema->format == NULL) {
1059
+ return ENOMEM;
1060
+ }
1061
+
1062
+ memcpy((void*)schema->format, format, format_size);
1063
+ } else {
1064
+ schema->format = NULL;
1065
+ }
1066
+
1067
+ return NANOARROW_OK;
1068
+ }
1069
+
1070
+ ArrowErrorCode ArrowSchemaSetName(struct ArrowSchema* schema, const char* name) {
1071
+ if (schema->name != NULL) {
1072
+ ArrowFree((void*)schema->name);
1073
+ }
1074
+
1075
+ if (name != NULL) {
1076
+ size_t name_size = strlen(name) + 1;
1077
+ schema->name = (const char*)ArrowMalloc(name_size);
1078
+ if (schema->name == NULL) {
1079
+ return ENOMEM;
1080
+ }
1081
+
1082
+ memcpy((void*)schema->name, name, name_size);
1083
+ } else {
1084
+ schema->name = NULL;
1085
+ }
1086
+
1087
+ return NANOARROW_OK;
1088
+ }
1089
+
1090
+ ArrowErrorCode ArrowSchemaSetMetadata(struct ArrowSchema* schema, const char* metadata) {
1091
+ if (schema->metadata != NULL) {
1092
+ ArrowFree((void*)schema->metadata);
1093
+ }
1094
+
1095
+ if (metadata != NULL) {
1096
+ size_t metadata_size = ArrowMetadataSizeOf(metadata);
1097
+ schema->metadata = (const char*)ArrowMalloc(metadata_size);
1098
+ if (schema->metadata == NULL) {
1099
+ return ENOMEM;
1100
+ }
1101
+
1102
+ memcpy((void*)schema->metadata, metadata, metadata_size);
1103
+ } else {
1104
+ schema->metadata = NULL;
1105
+ }
1106
+
1107
+ return NANOARROW_OK;
1108
+ }
1109
+
1110
+ ArrowErrorCode ArrowSchemaAllocateChildren(struct ArrowSchema* schema,
1111
+ int64_t n_children) {
1112
+ if (schema->children != NULL) {
1113
+ return EEXIST;
1114
+ }
1115
+
1116
+ if (n_children > 0) {
1117
+ schema->children =
1118
+ (struct ArrowSchema**)ArrowMalloc(n_children * sizeof(struct ArrowSchema*));
1119
+
1120
+ if (schema->children == NULL) {
1121
+ return ENOMEM;
1122
+ }
1123
+
1124
+ schema->n_children = n_children;
1125
+
1126
+ memset(schema->children, 0, n_children * sizeof(struct ArrowSchema*));
1127
+
1128
+ for (int64_t i = 0; i < n_children; i++) {
1129
+ schema->children[i] = (struct ArrowSchema*)ArrowMalloc(sizeof(struct ArrowSchema));
1130
+
1131
+ if (schema->children[i] == NULL) {
1132
+ return ENOMEM;
1133
+ }
1134
+
1135
+ schema->children[i]->release = NULL;
1136
+ }
1137
+ }
1138
+
1139
+ return NANOARROW_OK;
1140
+ }
1141
+
1142
+ ArrowErrorCode ArrowSchemaAllocateDictionary(struct ArrowSchema* schema) {
1143
+ if (schema->dictionary != NULL) {
1144
+ return EEXIST;
1145
+ }
1146
+
1147
+ schema->dictionary = (struct ArrowSchema*)ArrowMalloc(sizeof(struct ArrowSchema));
1148
+ if (schema->dictionary == NULL) {
1149
+ return ENOMEM;
1150
+ }
1151
+
1152
+ schema->dictionary->release = NULL;
1153
+ return NANOARROW_OK;
1154
+ }
1155
+
1156
+ ArrowErrorCode ArrowSchemaDeepCopy(const struct ArrowSchema* schema,
1157
+ struct ArrowSchema* schema_out) {
1158
+ ArrowSchemaInit(schema_out);
1159
+
1160
+ int result = ArrowSchemaSetFormat(schema_out, schema->format);
1161
+ if (result != NANOARROW_OK) {
1162
+ ArrowSchemaRelease(schema_out);
1163
+ return result;
1164
+ }
1165
+
1166
+ schema_out->flags = schema->flags;
1167
+
1168
+ result = ArrowSchemaSetName(schema_out, schema->name);
1169
+ if (result != NANOARROW_OK) {
1170
+ ArrowSchemaRelease(schema_out);
1171
+ return result;
1172
+ }
1173
+
1174
+ result = ArrowSchemaSetMetadata(schema_out, schema->metadata);
1175
+ if (result != NANOARROW_OK) {
1176
+ ArrowSchemaRelease(schema_out);
1177
+ return result;
1178
+ }
1179
+
1180
+ result = ArrowSchemaAllocateChildren(schema_out, schema->n_children);
1181
+ if (result != NANOARROW_OK) {
1182
+ ArrowSchemaRelease(schema_out);
1183
+ return result;
1184
+ }
1185
+
1186
+ for (int64_t i = 0; i < schema->n_children; i++) {
1187
+ result = ArrowSchemaDeepCopy(schema->children[i], schema_out->children[i]);
1188
+ if (result != NANOARROW_OK) {
1189
+ ArrowSchemaRelease(schema_out);
1190
+ return result;
1191
+ }
1192
+ }
1193
+
1194
+ if (schema->dictionary != NULL) {
1195
+ result = ArrowSchemaAllocateDictionary(schema_out);
1196
+ if (result != NANOARROW_OK) {
1197
+ ArrowSchemaRelease(schema_out);
1198
+ return result;
1199
+ }
1200
+
1201
+ result = ArrowSchemaDeepCopy(schema->dictionary, schema_out->dictionary);
1202
+ if (result != NANOARROW_OK) {
1203
+ ArrowSchemaRelease(schema_out);
1204
+ return result;
1205
+ }
1206
+ }
1207
+
1208
+ return NANOARROW_OK;
1209
+ }
1210
+
1211
+ static void ArrowSchemaViewSetPrimitive(struct ArrowSchemaView* schema_view,
1212
+ enum ArrowType type) {
1213
+ schema_view->type = type;
1214
+ schema_view->storage_type = type;
1215
+ }
1216
+
1217
+ static ArrowErrorCode ArrowSchemaViewParse(struct ArrowSchemaView* schema_view,
1218
+ const char* format,
1219
+ const char** format_end_out,
1220
+ struct ArrowError* error) {
1221
+ *format_end_out = format;
1222
+
1223
+ // needed for decimal parsing
1224
+ const char* parse_start;
1225
+ char* parse_end;
1226
+
1227
+ switch (format[0]) {
1228
+ case 'n':
1229
+ schema_view->type = NANOARROW_TYPE_NA;
1230
+ schema_view->storage_type = NANOARROW_TYPE_NA;
1231
+ *format_end_out = format + 1;
1232
+ return NANOARROW_OK;
1233
+ case 'b':
1234
+ ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_BOOL);
1235
+ *format_end_out = format + 1;
1236
+ return NANOARROW_OK;
1237
+ case 'c':
1238
+ ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INT8);
1239
+ *format_end_out = format + 1;
1240
+ return NANOARROW_OK;
1241
+ case 'C':
1242
+ ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_UINT8);
1243
+ *format_end_out = format + 1;
1244
+ return NANOARROW_OK;
1245
+ case 's':
1246
+ ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INT16);
1247
+ *format_end_out = format + 1;
1248
+ return NANOARROW_OK;
1249
+ case 'S':
1250
+ ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_UINT16);
1251
+ *format_end_out = format + 1;
1252
+ return NANOARROW_OK;
1253
+ case 'i':
1254
+ ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INT32);
1255
+ *format_end_out = format + 1;
1256
+ return NANOARROW_OK;
1257
+ case 'I':
1258
+ ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_UINT32);
1259
+ *format_end_out = format + 1;
1260
+ return NANOARROW_OK;
1261
+ case 'l':
1262
+ ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INT64);
1263
+ *format_end_out = format + 1;
1264
+ return NANOARROW_OK;
1265
+ case 'L':
1266
+ ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_UINT64);
1267
+ *format_end_out = format + 1;
1268
+ return NANOARROW_OK;
1269
+ case 'e':
1270
+ ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_HALF_FLOAT);
1271
+ *format_end_out = format + 1;
1272
+ return NANOARROW_OK;
1273
+ case 'f':
1274
+ ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_FLOAT);
1275
+ *format_end_out = format + 1;
1276
+ return NANOARROW_OK;
1277
+ case 'g':
1278
+ ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_DOUBLE);
1279
+ *format_end_out = format + 1;
1280
+ return NANOARROW_OK;
1281
+
1282
+ // decimal
1283
+ case 'd':
1284
+ if (format[1] != ':' || format[2] == '\0') {
1285
+ ArrowErrorSet(error, "Expected ':precision,scale[,bitwidth]' following 'd'");
1286
+ return EINVAL;
1287
+ }
1288
+
1289
+ parse_start = format + 2;
1290
+ schema_view->decimal_precision = (int32_t)strtol(parse_start, &parse_end, 10);
1291
+ if (parse_end == parse_start || parse_end[0] != ',') {
1292
+ ArrowErrorSet(error, "Expected 'precision,scale[,bitwidth]' following 'd:'");
1293
+ return EINVAL;
1294
+ }
1295
+
1296
+ parse_start = parse_end + 1;
1297
+ schema_view->decimal_scale = (int32_t)strtol(parse_start, &parse_end, 10);
1298
+ if (parse_end == parse_start) {
1299
+ ArrowErrorSet(error, "Expected 'scale[,bitwidth]' following 'd:precision,'");
1300
+ return EINVAL;
1301
+ } else if (parse_end[0] != ',') {
1302
+ schema_view->decimal_bitwidth = 128;
1303
+ } else {
1304
+ parse_start = parse_end + 1;
1305
+ schema_view->decimal_bitwidth = (int32_t)strtol(parse_start, &parse_end, 10);
1306
+ if (parse_start == parse_end) {
1307
+ ArrowErrorSet(error, "Expected precision following 'd:precision,scale,'");
1308
+ return EINVAL;
1309
+ }
1310
+ }
1311
+
1312
+ *format_end_out = parse_end;
1313
+
1314
+ switch (schema_view->decimal_bitwidth) {
1315
+ case 32:
1316
+ ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_DECIMAL32);
1317
+ return NANOARROW_OK;
1318
+ case 64:
1319
+ ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_DECIMAL64);
1320
+ return NANOARROW_OK;
1321
+ case 128:
1322
+ ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_DECIMAL128);
1323
+ return NANOARROW_OK;
1324
+ case 256:
1325
+ ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_DECIMAL256);
1326
+ return NANOARROW_OK;
1327
+ default:
1328
+ ArrowErrorSet(error,
1329
+ "Expected decimal bitwidth of 128 or 256 but found %" PRId32,
1330
+ schema_view->decimal_bitwidth);
1331
+ return EINVAL;
1332
+ }
1333
+
1334
+ // validity + data
1335
+ case 'w':
1336
+ schema_view->type = NANOARROW_TYPE_FIXED_SIZE_BINARY;
1337
+ schema_view->storage_type = NANOARROW_TYPE_FIXED_SIZE_BINARY;
1338
+ if (format[1] != ':' || format[2] == '\0') {
1339
+ ArrowErrorSet(error, "Expected ':<width>' following 'w'");
1340
+ return EINVAL;
1341
+ }
1342
+
1343
+ schema_view->fixed_size = (int32_t)strtol(format + 2, (char**)format_end_out, 10);
1344
+ return NANOARROW_OK;
1345
+
1346
+ // validity + offset + data
1347
+ case 'z':
1348
+ schema_view->type = NANOARROW_TYPE_BINARY;
1349
+ schema_view->storage_type = NANOARROW_TYPE_BINARY;
1350
+ *format_end_out = format + 1;
1351
+ return NANOARROW_OK;
1352
+ case 'u':
1353
+ schema_view->type = NANOARROW_TYPE_STRING;
1354
+ schema_view->storage_type = NANOARROW_TYPE_STRING;
1355
+ *format_end_out = format + 1;
1356
+ return NANOARROW_OK;
1357
+
1358
+ // validity + large_offset + data
1359
+ case 'Z':
1360
+ schema_view->type = NANOARROW_TYPE_LARGE_BINARY;
1361
+ schema_view->storage_type = NANOARROW_TYPE_LARGE_BINARY;
1362
+ *format_end_out = format + 1;
1363
+ return NANOARROW_OK;
1364
+ case 'U':
1365
+ schema_view->type = NANOARROW_TYPE_LARGE_STRING;
1366
+ schema_view->storage_type = NANOARROW_TYPE_LARGE_STRING;
1367
+ *format_end_out = format + 1;
1368
+ return NANOARROW_OK;
1369
+
1370
+ // nested types
1371
+ case '+':
1372
+ switch (format[1]) {
1373
+ // list has validity + offset or offset
1374
+ case 'l':
1375
+ schema_view->storage_type = NANOARROW_TYPE_LIST;
1376
+ schema_view->type = NANOARROW_TYPE_LIST;
1377
+ *format_end_out = format + 2;
1378
+ return NANOARROW_OK;
1379
+
1380
+ // large list has validity + large_offset or large_offset
1381
+ case 'L':
1382
+ schema_view->storage_type = NANOARROW_TYPE_LARGE_LIST;
1383
+ schema_view->type = NANOARROW_TYPE_LARGE_LIST;
1384
+ *format_end_out = format + 2;
1385
+ return NANOARROW_OK;
1386
+
1387
+ // run end encoded has no buffer at all
1388
+ case 'r':
1389
+ schema_view->storage_type = NANOARROW_TYPE_RUN_END_ENCODED;
1390
+ schema_view->type = NANOARROW_TYPE_RUN_END_ENCODED;
1391
+ *format_end_out = format + 2;
1392
+ return NANOARROW_OK;
1393
+
1394
+ // just validity buffer
1395
+ case 'w':
1396
+ if (format[2] != ':' || format[3] == '\0') {
1397
+ ArrowErrorSet(error, "Expected ':<width>' following '+w'");
1398
+ return EINVAL;
1399
+ }
1400
+
1401
+ schema_view->storage_type = NANOARROW_TYPE_FIXED_SIZE_LIST;
1402
+ schema_view->type = NANOARROW_TYPE_FIXED_SIZE_LIST;
1403
+ schema_view->fixed_size =
1404
+ (int32_t)strtol(format + 3, (char**)format_end_out, 10);
1405
+ return NANOARROW_OK;
1406
+ case 's':
1407
+ schema_view->storage_type = NANOARROW_TYPE_STRUCT;
1408
+ schema_view->type = NANOARROW_TYPE_STRUCT;
1409
+ *format_end_out = format + 2;
1410
+ return NANOARROW_OK;
1411
+ case 'm':
1412
+ schema_view->storage_type = NANOARROW_TYPE_MAP;
1413
+ schema_view->type = NANOARROW_TYPE_MAP;
1414
+ *format_end_out = format + 2;
1415
+ return NANOARROW_OK;
1416
+
1417
+ // unions
1418
+ case 'u':
1419
+ switch (format[2]) {
1420
+ case 'd':
1421
+ schema_view->storage_type = NANOARROW_TYPE_DENSE_UNION;
1422
+ schema_view->type = NANOARROW_TYPE_DENSE_UNION;
1423
+ break;
1424
+ case 's':
1425
+ schema_view->storage_type = NANOARROW_TYPE_SPARSE_UNION;
1426
+ schema_view->type = NANOARROW_TYPE_SPARSE_UNION;
1427
+ break;
1428
+ default:
1429
+ ArrowErrorSet(error,
1430
+ "Expected union format string +us:<type_ids> or "
1431
+ "+ud:<type_ids> but found '%s'",
1432
+ format);
1433
+ return EINVAL;
1434
+ }
1435
+
1436
+ if (format[3] == ':') {
1437
+ schema_view->union_type_ids = format + 4;
1438
+ int64_t n_type_ids =
1439
+ _ArrowParseUnionTypeIds(schema_view->union_type_ids, NULL);
1440
+ if (n_type_ids != schema_view->schema->n_children) {
1441
+ ArrowErrorSet(error,
1442
+ "Expected union type_ids parameter to be a comma-separated "
1443
+ "list of %" PRId64 " values between 0 and 127 but found '%s'",
1444
+ schema_view->schema->n_children, schema_view->union_type_ids);
1445
+ return EINVAL;
1446
+ }
1447
+ *format_end_out = format + strlen(format);
1448
+ return NANOARROW_OK;
1449
+ } else {
1450
+ ArrowErrorSet(error,
1451
+ "Expected union format string +us:<type_ids> or +ud:<type_ids> "
1452
+ "but found '%s'",
1453
+ format);
1454
+ return EINVAL;
1455
+ }
1456
+
1457
+ // views
1458
+ case 'v':
1459
+ switch (format[2]) {
1460
+ case 'l':
1461
+ schema_view->storage_type = NANOARROW_TYPE_LIST_VIEW;
1462
+ schema_view->type = NANOARROW_TYPE_LIST_VIEW;
1463
+ *format_end_out = format + 3;
1464
+ return NANOARROW_OK;
1465
+ case 'L':
1466
+ schema_view->storage_type = NANOARROW_TYPE_LARGE_LIST_VIEW;
1467
+ schema_view->type = NANOARROW_TYPE_LARGE_LIST_VIEW;
1468
+ *format_end_out = format + 3;
1469
+ return NANOARROW_OK;
1470
+ default:
1471
+ ArrowErrorSet(
1472
+ error, "Expected view format string +vl or +vL but found '%s'", format);
1473
+ return EINVAL;
1474
+ }
1475
+ default:
1476
+ ArrowErrorSet(error, "Expected nested type format string but found '%s'",
1477
+ format);
1478
+ return EINVAL;
1479
+ }
1480
+
1481
+ // date/time types
1482
+ case 't':
1483
+ switch (format[1]) {
1484
+ // date
1485
+ case 'd':
1486
+ switch (format[2]) {
1487
+ case 'D':
1488
+ ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INT32);
1489
+ schema_view->type = NANOARROW_TYPE_DATE32;
1490
+ *format_end_out = format + 3;
1491
+ return NANOARROW_OK;
1492
+ case 'm':
1493
+ ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INT64);
1494
+ schema_view->type = NANOARROW_TYPE_DATE64;
1495
+ *format_end_out = format + 3;
1496
+ return NANOARROW_OK;
1497
+ default:
1498
+ ArrowErrorSet(error, "Expected 'D' or 'm' following 'td' but found '%s'",
1499
+ format + 2);
1500
+ return EINVAL;
1501
+ }
1502
+
1503
+ // time of day
1504
+ case 't':
1505
+ switch (format[2]) {
1506
+ case 's':
1507
+ ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INT32);
1508
+ schema_view->type = NANOARROW_TYPE_TIME32;
1509
+ schema_view->time_unit = NANOARROW_TIME_UNIT_SECOND;
1510
+ *format_end_out = format + 3;
1511
+ return NANOARROW_OK;
1512
+ case 'm':
1513
+ ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INT32);
1514
+ schema_view->type = NANOARROW_TYPE_TIME32;
1515
+ schema_view->time_unit = NANOARROW_TIME_UNIT_MILLI;
1516
+ *format_end_out = format + 3;
1517
+ return NANOARROW_OK;
1518
+ case 'u':
1519
+ ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INT64);
1520
+ schema_view->type = NANOARROW_TYPE_TIME64;
1521
+ schema_view->time_unit = NANOARROW_TIME_UNIT_MICRO;
1522
+ *format_end_out = format + 3;
1523
+ return NANOARROW_OK;
1524
+ case 'n':
1525
+ ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INT64);
1526
+ schema_view->type = NANOARROW_TYPE_TIME64;
1527
+ schema_view->time_unit = NANOARROW_TIME_UNIT_NANO;
1528
+ *format_end_out = format + 3;
1529
+ return NANOARROW_OK;
1530
+ default:
1531
+ ArrowErrorSet(
1532
+ error, "Expected 's', 'm', 'u', or 'n' following 'tt' but found '%s'",
1533
+ format + 2);
1534
+ return EINVAL;
1535
+ }
1536
+
1537
+ // timestamp
1538
+ case 's':
1539
+ switch (format[2]) {
1540
+ case 's':
1541
+ ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INT64);
1542
+ schema_view->type = NANOARROW_TYPE_TIMESTAMP;
1543
+ schema_view->time_unit = NANOARROW_TIME_UNIT_SECOND;
1544
+ break;
1545
+ case 'm':
1546
+ ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INT64);
1547
+ schema_view->type = NANOARROW_TYPE_TIMESTAMP;
1548
+ schema_view->time_unit = NANOARROW_TIME_UNIT_MILLI;
1549
+ break;
1550
+ case 'u':
1551
+ ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INT64);
1552
+ schema_view->type = NANOARROW_TYPE_TIMESTAMP;
1553
+ schema_view->time_unit = NANOARROW_TIME_UNIT_MICRO;
1554
+ break;
1555
+ case 'n':
1556
+ ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INT64);
1557
+ schema_view->type = NANOARROW_TYPE_TIMESTAMP;
1558
+ schema_view->time_unit = NANOARROW_TIME_UNIT_NANO;
1559
+ break;
1560
+ default:
1561
+ ArrowErrorSet(
1562
+ error, "Expected 's', 'm', 'u', or 'n' following 'ts' but found '%s'",
1563
+ format + 2);
1564
+ return EINVAL;
1565
+ }
1566
+
1567
+ if (format[3] != ':') {
1568
+ ArrowErrorSet(error, "Expected ':' following '%.3s' but found '%s'", format,
1569
+ format + 3);
1570
+ return EINVAL;
1571
+ }
1572
+
1573
+ schema_view->timezone = format + 4;
1574
+ *format_end_out = format + strlen(format);
1575
+ return NANOARROW_OK;
1576
+
1577
+ // duration
1578
+ case 'D':
1579
+ switch (format[2]) {
1580
+ case 's':
1581
+ ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INT64);
1582
+ schema_view->type = NANOARROW_TYPE_DURATION;
1583
+ schema_view->time_unit = NANOARROW_TIME_UNIT_SECOND;
1584
+ *format_end_out = format + 3;
1585
+ return NANOARROW_OK;
1586
+ case 'm':
1587
+ ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INT64);
1588
+ schema_view->type = NANOARROW_TYPE_DURATION;
1589
+ schema_view->time_unit = NANOARROW_TIME_UNIT_MILLI;
1590
+ *format_end_out = format + 3;
1591
+ return NANOARROW_OK;
1592
+ case 'u':
1593
+ ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INT64);
1594
+ schema_view->type = NANOARROW_TYPE_DURATION;
1595
+ schema_view->time_unit = NANOARROW_TIME_UNIT_MICRO;
1596
+ *format_end_out = format + 3;
1597
+ return NANOARROW_OK;
1598
+ case 'n':
1599
+ ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INT64);
1600
+ schema_view->type = NANOARROW_TYPE_DURATION;
1601
+ schema_view->time_unit = NANOARROW_TIME_UNIT_NANO;
1602
+ *format_end_out = format + 3;
1603
+ return NANOARROW_OK;
1604
+ default:
1605
+ ArrowErrorSet(error,
1606
+ "Expected 's', 'm', u', or 'n' following 'tD' but found '%s'",
1607
+ format + 2);
1608
+ return EINVAL;
1609
+ }
1610
+
1611
+ // interval
1612
+ case 'i':
1613
+ switch (format[2]) {
1614
+ case 'M':
1615
+ ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INTERVAL_MONTHS);
1616
+ *format_end_out = format + 3;
1617
+ return NANOARROW_OK;
1618
+ case 'D':
1619
+ ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INTERVAL_DAY_TIME);
1620
+ *format_end_out = format + 3;
1621
+ return NANOARROW_OK;
1622
+ case 'n':
1623
+ ArrowSchemaViewSetPrimitive(schema_view,
1624
+ NANOARROW_TYPE_INTERVAL_MONTH_DAY_NANO);
1625
+ *format_end_out = format + 3;
1626
+ return NANOARROW_OK;
1627
+ default:
1628
+ ArrowErrorSet(error,
1629
+ "Expected 'M', 'D', or 'n' following 'ti' but found '%s'",
1630
+ format + 2);
1631
+ return EINVAL;
1632
+ }
1633
+
1634
+ default:
1635
+ ArrowErrorSet(
1636
+ error, "Expected 'd', 't', 's', 'D', or 'i' following 't' but found '%s'",
1637
+ format + 1);
1638
+ return EINVAL;
1639
+ }
1640
+
1641
+ // view types
1642
+ case 'v': {
1643
+ switch (format[1]) {
1644
+ case 'u':
1645
+ ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_STRING_VIEW);
1646
+ *format_end_out = format + 2;
1647
+ return NANOARROW_OK;
1648
+ case 'z':
1649
+ ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_BINARY_VIEW);
1650
+ *format_end_out = format + 2;
1651
+ return NANOARROW_OK;
1652
+ default:
1653
+ ArrowErrorSet(error, "Expected 'u', or 'z' following 'v' but found '%s'",
1654
+ format + 1);
1655
+ return EINVAL;
1656
+ }
1657
+ }
1658
+
1659
+ default:
1660
+ ArrowErrorSet(error, "Unknown format: '%s'", format);
1661
+ return EINVAL;
1662
+ }
1663
+ }
1664
+
1665
+ static ArrowErrorCode ArrowSchemaViewValidateNChildren(
1666
+ struct ArrowSchemaView* schema_view, int64_t n_children, struct ArrowError* error) {
1667
+ if (n_children != -1 && schema_view->schema->n_children != n_children) {
1668
+ ArrowErrorSet(
1669
+ error, "Expected schema with %" PRId64 " children but found %" PRId64 " children",
1670
+ n_children, schema_view->schema->n_children);
1671
+ return EINVAL;
1672
+ }
1673
+
1674
+ // Don't do a full validation of children but do check that they won't
1675
+ // segfault if inspected
1676
+ struct ArrowSchema* child;
1677
+ for (int64_t i = 0; i < schema_view->schema->n_children; i++) {
1678
+ child = schema_view->schema->children[i];
1679
+ if (child == NULL) {
1680
+ ArrowErrorSet(
1681
+ error, "Expected valid schema at schema->children[%" PRId64 "] but found NULL",
1682
+ i);
1683
+ return EINVAL;
1684
+ } else if (child->release == NULL) {
1685
+ ArrowErrorSet(error,
1686
+ "Expected valid schema at schema->children[%" PRId64
1687
+ "] but found a released schema",
1688
+ i);
1689
+ return EINVAL;
1690
+ }
1691
+ }
1692
+
1693
+ return NANOARROW_OK;
1694
+ }
1695
+
1696
+ static ArrowErrorCode ArrowSchemaViewValidateUnion(struct ArrowSchemaView* schema_view,
1697
+ struct ArrowError* error) {
1698
+ return ArrowSchemaViewValidateNChildren(schema_view, -1, error);
1699
+ }
1700
+
1701
+ static ArrowErrorCode ArrowSchemaViewValidateMap(struct ArrowSchemaView* schema_view,
1702
+ struct ArrowError* error) {
1703
+ NANOARROW_RETURN_NOT_OK(ArrowSchemaViewValidateNChildren(schema_view, 1, error));
1704
+
1705
+ if (schema_view->schema->children[0]->n_children != 2) {
1706
+ ArrowErrorSet(error,
1707
+ "Expected child of map type to have 2 children but found %" PRId64,
1708
+ schema_view->schema->children[0]->n_children);
1709
+ return EINVAL;
1710
+ }
1711
+
1712
+ if (strcmp(schema_view->schema->children[0]->format, "+s") != 0) {
1713
+ ArrowErrorSet(error, "Expected format of child of map type to be '+s' but found '%s'",
1714
+ schema_view->schema->children[0]->format);
1715
+ return EINVAL;
1716
+ }
1717
+
1718
+ if (schema_view->schema->children[0]->flags & ARROW_FLAG_NULLABLE) {
1719
+ ArrowErrorSet(error,
1720
+ "Expected child of map type to be non-nullable but was nullable");
1721
+ return EINVAL;
1722
+ }
1723
+
1724
+ if (schema_view->schema->children[0]->children[0]->flags & ARROW_FLAG_NULLABLE) {
1725
+ ArrowErrorSet(error, "Expected key of map type to be non-nullable but was nullable");
1726
+ return EINVAL;
1727
+ }
1728
+
1729
+ return NANOARROW_OK;
1730
+ }
1731
+
1732
+ static ArrowErrorCode ArrowSchemaViewValidateDictionary(
1733
+ struct ArrowSchemaView* schema_view, struct ArrowError* error) {
1734
+ // check for valid index type
1735
+ switch (schema_view->storage_type) {
1736
+ case NANOARROW_TYPE_UINT8:
1737
+ case NANOARROW_TYPE_INT8:
1738
+ case NANOARROW_TYPE_UINT16:
1739
+ case NANOARROW_TYPE_INT16:
1740
+ case NANOARROW_TYPE_UINT32:
1741
+ case NANOARROW_TYPE_INT32:
1742
+ case NANOARROW_TYPE_UINT64:
1743
+ case NANOARROW_TYPE_INT64:
1744
+ break;
1745
+ default:
1746
+ ArrowErrorSet(
1747
+ error,
1748
+ "Expected dictionary schema index type to be an integral type but found '%s'",
1749
+ schema_view->schema->format);
1750
+ return EINVAL;
1751
+ }
1752
+
1753
+ struct ArrowSchemaView dictionary_schema_view;
1754
+ return ArrowSchemaViewInit(&dictionary_schema_view, schema_view->schema->dictionary,
1755
+ error);
1756
+ }
1757
+
1758
+ static ArrowErrorCode ArrowSchemaViewValidate(struct ArrowSchemaView* schema_view,
1759
+ enum ArrowType type,
1760
+ struct ArrowError* error) {
1761
+ switch (type) {
1762
+ case NANOARROW_TYPE_NA:
1763
+ case NANOARROW_TYPE_BOOL:
1764
+ case NANOARROW_TYPE_UINT8:
1765
+ case NANOARROW_TYPE_INT8:
1766
+ case NANOARROW_TYPE_UINT16:
1767
+ case NANOARROW_TYPE_INT16:
1768
+ case NANOARROW_TYPE_UINT32:
1769
+ case NANOARROW_TYPE_INT32:
1770
+ case NANOARROW_TYPE_UINT64:
1771
+ case NANOARROW_TYPE_INT64:
1772
+ case NANOARROW_TYPE_HALF_FLOAT:
1773
+ case NANOARROW_TYPE_FLOAT:
1774
+ case NANOARROW_TYPE_DOUBLE:
1775
+ case NANOARROW_TYPE_DECIMAL32:
1776
+ case NANOARROW_TYPE_DECIMAL64:
1777
+ case NANOARROW_TYPE_DECIMAL128:
1778
+ case NANOARROW_TYPE_DECIMAL256:
1779
+ case NANOARROW_TYPE_STRING:
1780
+ case NANOARROW_TYPE_LARGE_STRING:
1781
+ case NANOARROW_TYPE_BINARY:
1782
+ case NANOARROW_TYPE_LARGE_BINARY:
1783
+ case NANOARROW_TYPE_DATE32:
1784
+ case NANOARROW_TYPE_DATE64:
1785
+ case NANOARROW_TYPE_INTERVAL_MONTHS:
1786
+ case NANOARROW_TYPE_INTERVAL_DAY_TIME:
1787
+ case NANOARROW_TYPE_INTERVAL_MONTH_DAY_NANO:
1788
+ case NANOARROW_TYPE_TIMESTAMP:
1789
+ case NANOARROW_TYPE_TIME32:
1790
+ case NANOARROW_TYPE_TIME64:
1791
+ case NANOARROW_TYPE_DURATION:
1792
+ case NANOARROW_TYPE_BINARY_VIEW:
1793
+ case NANOARROW_TYPE_STRING_VIEW:
1794
+ return ArrowSchemaViewValidateNChildren(schema_view, 0, error);
1795
+
1796
+ case NANOARROW_TYPE_FIXED_SIZE_BINARY:
1797
+ if (schema_view->fixed_size <= 0) {
1798
+ ArrowErrorSet(error, "Expected size > 0 for fixed size binary but found size %d",
1799
+ schema_view->fixed_size);
1800
+ return EINVAL;
1801
+ }
1802
+ return ArrowSchemaViewValidateNChildren(schema_view, 0, error);
1803
+
1804
+ case NANOARROW_TYPE_LIST:
1805
+ case NANOARROW_TYPE_LIST_VIEW:
1806
+ case NANOARROW_TYPE_LARGE_LIST:
1807
+ case NANOARROW_TYPE_LARGE_LIST_VIEW:
1808
+ case NANOARROW_TYPE_FIXED_SIZE_LIST:
1809
+ return ArrowSchemaViewValidateNChildren(schema_view, 1, error);
1810
+
1811
+ case NANOARROW_TYPE_RUN_END_ENCODED:
1812
+ return ArrowSchemaViewValidateNChildren(schema_view, 2, error);
1813
+
1814
+ case NANOARROW_TYPE_STRUCT:
1815
+ return ArrowSchemaViewValidateNChildren(schema_view, -1, error);
1816
+
1817
+ case NANOARROW_TYPE_SPARSE_UNION:
1818
+ case NANOARROW_TYPE_DENSE_UNION:
1819
+ return ArrowSchemaViewValidateUnion(schema_view, error);
1820
+
1821
+ case NANOARROW_TYPE_MAP:
1822
+ return ArrowSchemaViewValidateMap(schema_view, error);
1823
+
1824
+ case NANOARROW_TYPE_DICTIONARY:
1825
+ return ArrowSchemaViewValidateDictionary(schema_view, error);
1826
+
1827
+ default:
1828
+ ArrowErrorSet(error, "Expected a valid enum ArrowType value but found %d",
1829
+ schema_view->type);
1830
+ return EINVAL;
1831
+ }
1832
+
1833
+ return NANOARROW_OK;
1834
+ }
1835
+
1836
+ ArrowErrorCode ArrowSchemaViewInit(struct ArrowSchemaView* schema_view,
1837
+ const struct ArrowSchema* schema,
1838
+ struct ArrowError* error) {
1839
+ if (schema == NULL) {
1840
+ ArrowErrorSet(error, "Expected non-NULL schema");
1841
+ return EINVAL;
1842
+ }
1843
+
1844
+ if (schema->release == NULL) {
1845
+ ArrowErrorSet(error, "Expected non-released schema");
1846
+ return EINVAL;
1847
+ }
1848
+
1849
+ schema_view->schema = schema;
1850
+
1851
+ const char* format = schema->format;
1852
+ if (format == NULL) {
1853
+ ArrowErrorSet(
1854
+ error,
1855
+ "Error parsing schema->format: Expected a null-terminated string but found NULL");
1856
+ return EINVAL;
1857
+ }
1858
+
1859
+ size_t format_len = strlen(format);
1860
+ if (format_len == 0) {
1861
+ ArrowErrorSet(error, "Error parsing schema->format: Expected a string with size > 0");
1862
+ return EINVAL;
1863
+ }
1864
+
1865
+ const char* format_end_out;
1866
+ int result = ArrowSchemaViewParse(schema_view, format, &format_end_out, error);
1867
+
1868
+ if (result != NANOARROW_OK) {
1869
+ if (error != NULL) {
1870
+ char child_error[1024];
1871
+ memcpy(child_error, ArrowErrorMessage(error), 1024);
1872
+ ArrowErrorSet(error, "Error parsing schema->format: %s", child_error);
1873
+ }
1874
+
1875
+ return result;
1876
+ }
1877
+
1878
+ if ((format + format_len) != format_end_out) {
1879
+ ArrowErrorSet(error, "Error parsing schema->format '%s': parsed %d/%zu characters",
1880
+ format, (int)(format_end_out - format), format_len);
1881
+ return EINVAL;
1882
+ }
1883
+
1884
+ if (schema->dictionary != NULL) {
1885
+ schema_view->type = NANOARROW_TYPE_DICTIONARY;
1886
+ }
1887
+
1888
+ NANOARROW_RETURN_NOT_OK(
1889
+ ArrowSchemaViewValidate(schema_view, schema_view->storage_type, error));
1890
+
1891
+ if (schema_view->storage_type != schema_view->type) {
1892
+ NANOARROW_RETURN_NOT_OK(
1893
+ ArrowSchemaViewValidate(schema_view, schema_view->type, error));
1894
+ }
1895
+
1896
+ int64_t unknown_flags = schema->flags & ~NANOARROW_FLAG_ALL_SUPPORTED;
1897
+ if (unknown_flags != 0) {
1898
+ ArrowErrorSet(error, "Unknown ArrowSchema flag");
1899
+ return EINVAL;
1900
+ }
1901
+
1902
+ if (schema->flags & ARROW_FLAG_DICTIONARY_ORDERED &&
1903
+ schema_view->type != NANOARROW_TYPE_DICTIONARY) {
1904
+ ArrowErrorSet(error,
1905
+ "ARROW_FLAG_DICTIONARY_ORDERED is only relevant for dictionaries");
1906
+ return EINVAL;
1907
+ }
1908
+
1909
+ if (schema->flags & ARROW_FLAG_MAP_KEYS_SORTED &&
1910
+ schema_view->type != NANOARROW_TYPE_MAP) {
1911
+ ArrowErrorSet(error, "ARROW_FLAG_MAP_KEYS_SORTED is only relevant for a map type");
1912
+ return EINVAL;
1913
+ }
1914
+
1915
+ ArrowLayoutInit(&schema_view->layout, schema_view->storage_type);
1916
+ if (schema_view->storage_type == NANOARROW_TYPE_FIXED_SIZE_BINARY) {
1917
+ schema_view->layout.element_size_bits[1] = (int64_t)schema_view->fixed_size * 8;
1918
+ } else if (schema_view->storage_type == NANOARROW_TYPE_FIXED_SIZE_LIST) {
1919
+ schema_view->layout.child_size_elements = schema_view->fixed_size;
1920
+ }
1921
+
1922
+ schema_view->extension_name = ArrowCharView(NULL);
1923
+ schema_view->extension_metadata = ArrowCharView(NULL);
1924
+ NANOARROW_RETURN_NOT_OK(ArrowMetadataGetValue(schema->metadata,
1925
+ ArrowCharView("ARROW:extension:name"),
1926
+ &schema_view->extension_name));
1927
+ NANOARROW_RETURN_NOT_OK(ArrowMetadataGetValue(schema->metadata,
1928
+ ArrowCharView("ARROW:extension:metadata"),
1929
+ &schema_view->extension_metadata));
1930
+
1931
+ return NANOARROW_OK;
1932
+ }
1933
+
1934
+ static int64_t ArrowSchemaTypeToStringInternal(struct ArrowSchemaView* schema_view,
1935
+ char* out, int64_t n) {
1936
+ const char* type_string = ArrowTypeString(schema_view->type);
1937
+ switch (schema_view->type) {
1938
+ case NANOARROW_TYPE_DECIMAL32:
1939
+ case NANOARROW_TYPE_DECIMAL64:
1940
+ case NANOARROW_TYPE_DECIMAL128:
1941
+ case NANOARROW_TYPE_DECIMAL256:
1942
+ return snprintf(out, n, "%s(%" PRId32 ", %" PRId32 ")", type_string,
1943
+ schema_view->decimal_precision, schema_view->decimal_scale);
1944
+ case NANOARROW_TYPE_TIMESTAMP:
1945
+ return snprintf(out, n, "%s('%s', '%s')", type_string,
1946
+ ArrowTimeUnitString(schema_view->time_unit), schema_view->timezone);
1947
+ case NANOARROW_TYPE_TIME32:
1948
+ case NANOARROW_TYPE_TIME64:
1949
+ case NANOARROW_TYPE_DURATION:
1950
+ return snprintf(out, n, "%s('%s')", type_string,
1951
+ ArrowTimeUnitString(schema_view->time_unit));
1952
+ case NANOARROW_TYPE_FIXED_SIZE_BINARY:
1953
+ case NANOARROW_TYPE_FIXED_SIZE_LIST:
1954
+ return snprintf(out, n, "%s(%" PRId32 ")", type_string, schema_view->fixed_size);
1955
+ case NANOARROW_TYPE_SPARSE_UNION:
1956
+ case NANOARROW_TYPE_DENSE_UNION:
1957
+ return snprintf(out, n, "%s([%s])", type_string, schema_view->union_type_ids);
1958
+ default:
1959
+ return snprintf(out, n, "%s", type_string);
1960
+ }
1961
+ }
1962
+
1963
+ // Helper for bookkeeping to emulate sprintf()-like behaviour spread
1964
+ // among multiple sprintf calls.
1965
+ static inline void ArrowToStringLogChars(char** out, int64_t n_chars_last,
1966
+ int64_t* n_remaining, int64_t* n_chars) {
1967
+ // In the unlikely snprintf() returning a negative value (encoding error),
1968
+ // ensure the result won't cause an out-of-bounds access.
1969
+ if (n_chars_last < 0) {
1970
+ n_chars_last = 0;
1971
+ }
1972
+
1973
+ *n_chars += n_chars_last;
1974
+ *n_remaining -= n_chars_last;
1975
+
1976
+ // n_remaining is never less than 0
1977
+ if (*n_remaining < 0) {
1978
+ *n_remaining = 0;
1979
+ }
1980
+
1981
+ // Can't do math on a NULL pointer
1982
+ if (*out != NULL) {
1983
+ *out += n_chars_last;
1984
+ }
1985
+ }
1986
+
1987
+ int64_t ArrowSchemaToString(const struct ArrowSchema* schema, char* out, int64_t n,
1988
+ char recursive) {
1989
+ if (schema == NULL) {
1990
+ return snprintf(out, n, "[invalid: pointer is null]");
1991
+ }
1992
+
1993
+ if (schema->release == NULL) {
1994
+ return snprintf(out, n, "[invalid: schema is released]");
1995
+ }
1996
+
1997
+ struct ArrowSchemaView schema_view;
1998
+ struct ArrowError error;
1999
+
2000
+ if (ArrowSchemaViewInit(&schema_view, schema, &error) != NANOARROW_OK) {
2001
+ return snprintf(out, n, "[invalid: %s]", ArrowErrorMessage(&error));
2002
+ }
2003
+
2004
+ // Extension type and dictionary should include both the top-level type
2005
+ // and the storage type.
2006
+ int is_extension = schema_view.extension_name.size_bytes > 0;
2007
+ int is_dictionary = schema->dictionary != NULL;
2008
+ int64_t n_chars = 0;
2009
+ int64_t n_chars_last = 0;
2010
+
2011
+ // Uncommon but not technically impossible that both are true
2012
+ if (is_extension && is_dictionary) {
2013
+ n_chars_last = snprintf(
2014
+ out, n, "%.*s{dictionary(%s)<", (int)schema_view.extension_name.size_bytes,
2015
+ schema_view.extension_name.data, ArrowTypeString(schema_view.storage_type));
2016
+ } else if (is_extension) {
2017
+ n_chars_last = snprintf(out, n, "%.*s{", (int)schema_view.extension_name.size_bytes,
2018
+ schema_view.extension_name.data);
2019
+ } else if (is_dictionary) {
2020
+ n_chars_last =
2021
+ snprintf(out, n, "dictionary(%s)<", ArrowTypeString(schema_view.storage_type));
2022
+ }
2023
+
2024
+ ArrowToStringLogChars(&out, n_chars_last, &n, &n_chars);
2025
+
2026
+ if (!is_dictionary) {
2027
+ n_chars_last = ArrowSchemaTypeToStringInternal(&schema_view, out, n);
2028
+ } else {
2029
+ n_chars_last = ArrowSchemaToString(schema->dictionary, out, n, recursive);
2030
+ }
2031
+
2032
+ ArrowToStringLogChars(&out, n_chars_last, &n, &n_chars);
2033
+
2034
+ if (recursive && schema->format[0] == '+') {
2035
+ n_chars_last = snprintf(out, n, "<");
2036
+ ArrowToStringLogChars(&out, n_chars_last, &n, &n_chars);
2037
+
2038
+ for (int64_t i = 0; i < schema->n_children; i++) {
2039
+ if (i > 0) {
2040
+ n_chars_last = snprintf(out, n, ", ");
2041
+ ArrowToStringLogChars(&out, n_chars_last, &n, &n_chars);
2042
+ }
2043
+
2044
+ // ArrowSchemaToStringInternal() will validate the child and print the error,
2045
+ // but we need the name first
2046
+ if (schema->children[i] != NULL && schema->children[i]->release != NULL &&
2047
+ schema->children[i]->name != NULL) {
2048
+ n_chars_last = snprintf(out, n, "%s: ", schema->children[i]->name);
2049
+ ArrowToStringLogChars(&out, n_chars_last, &n, &n_chars);
2050
+ }
2051
+
2052
+ n_chars_last = ArrowSchemaToString(schema->children[i], out, n, recursive);
2053
+ ArrowToStringLogChars(&out, n_chars_last, &n, &n_chars);
2054
+ }
2055
+
2056
+ n_chars_last = snprintf(out, n, ">");
2057
+ ArrowToStringLogChars(&out, n_chars_last, &n, &n_chars);
2058
+ }
2059
+
2060
+ if (is_extension && is_dictionary) {
2061
+ n_chars += snprintf(out, n, ">}");
2062
+ } else if (is_extension) {
2063
+ n_chars += snprintf(out, n, "}");
2064
+ } else if (is_dictionary) {
2065
+ n_chars += snprintf(out, n, ">");
2066
+ }
2067
+
2068
+ // Ensure that we always return a positive result
2069
+ if (n_chars > 0) {
2070
+ return n_chars;
2071
+ } else {
2072
+ return 0;
2073
+ }
2074
+ }
2075
+
2076
+ ArrowErrorCode ArrowMetadataReaderInit(struct ArrowMetadataReader* reader,
2077
+ const char* metadata) {
2078
+ reader->metadata = metadata;
2079
+
2080
+ if (reader->metadata == NULL) {
2081
+ reader->offset = 0;
2082
+ reader->remaining_keys = 0;
2083
+ } else {
2084
+ memcpy(&reader->remaining_keys, reader->metadata, sizeof(int32_t));
2085
+ reader->offset = sizeof(int32_t);
2086
+ }
2087
+
2088
+ return NANOARROW_OK;
2089
+ }
2090
+
2091
+ ArrowErrorCode ArrowMetadataReaderRead(struct ArrowMetadataReader* reader,
2092
+ struct ArrowStringView* key_out,
2093
+ struct ArrowStringView* value_out) {
2094
+ if (reader->remaining_keys <= 0) {
2095
+ return EINVAL;
2096
+ }
2097
+
2098
+ int64_t pos = 0;
2099
+
2100
+ int32_t key_size;
2101
+ memcpy(&key_size, reader->metadata + reader->offset + pos, sizeof(int32_t));
2102
+ pos += sizeof(int32_t);
2103
+
2104
+ key_out->data = reader->metadata + reader->offset + pos;
2105
+ key_out->size_bytes = key_size;
2106
+ pos += key_size;
2107
+
2108
+ int32_t value_size;
2109
+ memcpy(&value_size, reader->metadata + reader->offset + pos, sizeof(int32_t));
2110
+ pos += sizeof(int32_t);
2111
+
2112
+ value_out->data = reader->metadata + reader->offset + pos;
2113
+ value_out->size_bytes = value_size;
2114
+ pos += value_size;
2115
+
2116
+ reader->offset += pos;
2117
+ reader->remaining_keys--;
2118
+ return NANOARROW_OK;
2119
+ }
2120
+
2121
+ int64_t ArrowMetadataSizeOf(const char* metadata) {
2122
+ if (metadata == NULL) {
2123
+ return 0;
2124
+ }
2125
+
2126
+ struct ArrowMetadataReader reader;
2127
+ struct ArrowStringView key;
2128
+ struct ArrowStringView value;
2129
+ if (ArrowMetadataReaderInit(&reader, metadata) != NANOARROW_OK) {
2130
+ return 0;
2131
+ }
2132
+
2133
+ int64_t size = sizeof(int32_t);
2134
+ while (ArrowMetadataReaderRead(&reader, &key, &value) == NANOARROW_OK) {
2135
+ size += sizeof(int32_t) + key.size_bytes + sizeof(int32_t) + value.size_bytes;
2136
+ }
2137
+
2138
+ return size;
2139
+ }
2140
+
2141
+ static ArrowErrorCode ArrowMetadataGetValueInternal(const char* metadata,
2142
+ struct ArrowStringView* key,
2143
+ struct ArrowStringView* value_out) {
2144
+ struct ArrowMetadataReader reader;
2145
+ struct ArrowStringView existing_key;
2146
+ struct ArrowStringView existing_value;
2147
+ NANOARROW_RETURN_NOT_OK(ArrowMetadataReaderInit(&reader, metadata));
2148
+
2149
+ while (ArrowMetadataReaderRead(&reader, &existing_key, &existing_value) ==
2150
+ NANOARROW_OK) {
2151
+ int key_equal = key->size_bytes == existing_key.size_bytes &&
2152
+ strncmp(key->data, existing_key.data, existing_key.size_bytes) == 0;
2153
+ if (key_equal) {
2154
+ value_out->data = existing_value.data;
2155
+ value_out->size_bytes = existing_value.size_bytes;
2156
+ break;
2157
+ }
2158
+ }
2159
+
2160
+ return NANOARROW_OK;
2161
+ }
2162
+
2163
+ ArrowErrorCode ArrowMetadataGetValue(const char* metadata, struct ArrowStringView key,
2164
+ struct ArrowStringView* value_out) {
2165
+ if (value_out == NULL) {
2166
+ return EINVAL;
2167
+ }
2168
+
2169
+ return ArrowMetadataGetValueInternal(metadata, &key, value_out);
2170
+ }
2171
+
2172
+ char ArrowMetadataHasKey(const char* metadata, struct ArrowStringView key) {
2173
+ struct ArrowStringView value = ArrowCharView(NULL);
2174
+ if (ArrowMetadataGetValue(metadata, key, &value) != NANOARROW_OK) {
2175
+ return 0;
2176
+ }
2177
+
2178
+ return value.data != NULL;
2179
+ }
2180
+
2181
+ ArrowErrorCode ArrowMetadataBuilderInit(struct ArrowBuffer* buffer,
2182
+ const char* metadata) {
2183
+ ArrowBufferInit(buffer);
2184
+ return ArrowBufferAppend(buffer, metadata, ArrowMetadataSizeOf(metadata));
2185
+ }
2186
+
2187
+ static ArrowErrorCode ArrowMetadataBuilderAppendInternal(struct ArrowBuffer* buffer,
2188
+ struct ArrowStringView* key,
2189
+ struct ArrowStringView* value) {
2190
+ if (value == NULL) {
2191
+ return NANOARROW_OK;
2192
+ }
2193
+
2194
+ if (buffer->capacity_bytes == 0) {
2195
+ NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(buffer, 0));
2196
+ }
2197
+
2198
+ if (((size_t)buffer->capacity_bytes) < sizeof(int32_t)) {
2199
+ return EINVAL;
2200
+ }
2201
+
2202
+ int32_t n_keys;
2203
+ memcpy(&n_keys, buffer->data, sizeof(int32_t));
2204
+
2205
+ int32_t key_size = (int32_t)key->size_bytes;
2206
+ int32_t value_size = (int32_t)value->size_bytes;
2207
+ NANOARROW_RETURN_NOT_OK(ArrowBufferReserve(
2208
+ buffer, sizeof(int32_t) + key_size + sizeof(int32_t) + value_size));
2209
+
2210
+ ArrowBufferAppendUnsafe(buffer, &key_size, sizeof(int32_t));
2211
+ ArrowBufferAppendUnsafe(buffer, key->data, key_size);
2212
+ ArrowBufferAppendUnsafe(buffer, &value_size, sizeof(int32_t));
2213
+ ArrowBufferAppendUnsafe(buffer, value->data, value_size);
2214
+
2215
+ n_keys++;
2216
+ memcpy(buffer->data, &n_keys, sizeof(int32_t));
2217
+
2218
+ return NANOARROW_OK;
2219
+ }
2220
+
2221
+ static ArrowErrorCode ArrowMetadataBuilderSetInternal(struct ArrowBuffer* buffer,
2222
+ struct ArrowStringView* key,
2223
+ struct ArrowStringView* value) {
2224
+ // Inspect the current value to see if we can avoid copying the buffer
2225
+ struct ArrowStringView current_value = ArrowCharView(NULL);
2226
+ NANOARROW_RETURN_NOT_OK(
2227
+ ArrowMetadataGetValueInternal((const char*)buffer->data, key, &current_value));
2228
+
2229
+ // The key should be removed but no key exists
2230
+ if (value == NULL && current_value.data == NULL) {
2231
+ return NANOARROW_OK;
2232
+ }
2233
+
2234
+ // The key/value can be appended because no key exists
2235
+ if (value != NULL && current_value.data == NULL) {
2236
+ return ArrowMetadataBuilderAppendInternal(buffer, key, value);
2237
+ }
2238
+
2239
+ struct ArrowMetadataReader reader;
2240
+ struct ArrowStringView existing_key;
2241
+ struct ArrowStringView existing_value;
2242
+ NANOARROW_RETURN_NOT_OK(ArrowMetadataReaderInit(&reader, (const char*)buffer->data));
2243
+
2244
+ struct ArrowBuffer new_buffer;
2245
+ NANOARROW_RETURN_NOT_OK(ArrowMetadataBuilderInit(&new_buffer, NULL));
2246
+
2247
+ while (reader.remaining_keys > 0) {
2248
+ int result = ArrowMetadataReaderRead(&reader, &existing_key, &existing_value);
2249
+ if (result != NANOARROW_OK) {
2250
+ ArrowBufferReset(&new_buffer);
2251
+ return result;
2252
+ }
2253
+
2254
+ if (key->size_bytes == existing_key.size_bytes &&
2255
+ strncmp((const char*)key->data, (const char*)existing_key.data,
2256
+ existing_key.size_bytes) == 0) {
2257
+ result = ArrowMetadataBuilderAppendInternal(&new_buffer, key, value);
2258
+ value = NULL;
2259
+ } else {
2260
+ result =
2261
+ ArrowMetadataBuilderAppendInternal(&new_buffer, &existing_key, &existing_value);
2262
+ }
2263
+
2264
+ if (result != NANOARROW_OK) {
2265
+ ArrowBufferReset(&new_buffer);
2266
+ return result;
2267
+ }
2268
+ }
2269
+
2270
+ ArrowBufferReset(buffer);
2271
+ ArrowBufferMove(&new_buffer, buffer);
2272
+ return NANOARROW_OK;
2273
+ }
2274
+
2275
+ ArrowErrorCode ArrowMetadataBuilderAppend(struct ArrowBuffer* buffer,
2276
+ struct ArrowStringView key,
2277
+ struct ArrowStringView value) {
2278
+ return ArrowMetadataBuilderAppendInternal(buffer, &key, &value);
2279
+ }
2280
+
2281
+ ArrowErrorCode ArrowMetadataBuilderSet(struct ArrowBuffer* buffer,
2282
+ struct ArrowStringView key,
2283
+ struct ArrowStringView value) {
2284
+ return ArrowMetadataBuilderSetInternal(buffer, &key, &value);
2285
+ }
2286
+
2287
+ ArrowErrorCode ArrowMetadataBuilderRemove(struct ArrowBuffer* buffer,
2288
+ struct ArrowStringView key) {
2289
+ return ArrowMetadataBuilderSetInternal(buffer, &key, NULL);
2290
+ }
2291
+ // Licensed to the Apache Software Foundation (ASF) under one
2292
+ // or more contributor license agreements. See the NOTICE file
2293
+ // distributed with this work for additional information
2294
+ // regarding copyright ownership. The ASF licenses this file
2295
+ // to you under the Apache License, Version 2.0 (the
2296
+ // "License"); you may not use this file except in compliance
2297
+ // with the License. You may obtain a copy of the License at
2298
+ //
2299
+ // http://www.apache.org/licenses/LICENSE-2.0
2300
+ //
2301
+ // Unless required by applicable law or agreed to in writing,
2302
+ // software distributed under the License is distributed on an
2303
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
2304
+ // KIND, either express or implied. See the License for the
2305
+ // specific language governing permissions and limitations
2306
+ // under the License.
2307
+
2308
+ #include <errno.h>
2309
+ #include <inttypes.h>
2310
+ #include <stdarg.h>
2311
+ #include <stdbool.h>
2312
+ #include <stdio.h>
2313
+ #include <stdlib.h>
2314
+ #include <string.h>
2315
+
2316
+ #include "nanoarrow/nanoarrow.h"
2317
+
2318
+ static void ArrowArrayReleaseInternal(struct ArrowArray* array) {
2319
+ // Release buffers held by this array
2320
+ struct ArrowArrayPrivateData* private_data =
2321
+ (struct ArrowArrayPrivateData*)array->private_data;
2322
+ if (private_data != NULL) {
2323
+ ArrowBitmapReset(&private_data->bitmap);
2324
+ ArrowBufferReset(&private_data->buffers[0]);
2325
+ ArrowBufferReset(&private_data->buffers[1]);
2326
+ ArrowFree(private_data->buffer_data);
2327
+ for (int32_t i = 0; i < private_data->n_variadic_buffers; ++i) {
2328
+ ArrowBufferReset(&private_data->variadic_buffers[i]);
2329
+ }
2330
+ ArrowFree(private_data->variadic_buffers);
2331
+ ArrowFree(private_data);
2332
+ }
2333
+
2334
+ // This object owns the memory for all the children, but those
2335
+ // children may have been generated elsewhere and might have
2336
+ // their own release() callback.
2337
+ if (array->children != NULL) {
2338
+ for (int64_t i = 0; i < array->n_children; i++) {
2339
+ if (array->children[i] != NULL) {
2340
+ if (array->children[i]->release != NULL) {
2341
+ ArrowArrayRelease(array->children[i]);
2342
+ }
2343
+
2344
+ ArrowFree(array->children[i]);
2345
+ }
2346
+ }
2347
+
2348
+ ArrowFree(array->children);
2349
+ }
2350
+
2351
+ // This object owns the memory for the dictionary but it
2352
+ // may have been generated somewhere else and have its own
2353
+ // release() callback.
2354
+ if (array->dictionary != NULL) {
2355
+ if (array->dictionary->release != NULL) {
2356
+ ArrowArrayRelease(array->dictionary);
2357
+ }
2358
+
2359
+ ArrowFree(array->dictionary);
2360
+ }
2361
+
2362
+ // Mark released
2363
+ array->release = NULL;
2364
+ }
2365
+
2366
+ static int ArrowArrayIsInternal(struct ArrowArray* array) {
2367
+ return array->release == &ArrowArrayReleaseInternal;
2368
+ }
2369
+
2370
+ static ArrowErrorCode ArrowArraySetStorageType(struct ArrowArray* array,
2371
+ enum ArrowType storage_type) {
2372
+ switch (storage_type) {
2373
+ case NANOARROW_TYPE_UNINITIALIZED:
2374
+ case NANOARROW_TYPE_NA:
2375
+ case NANOARROW_TYPE_RUN_END_ENCODED:
2376
+ array->n_buffers = 0;
2377
+ break;
2378
+
2379
+ case NANOARROW_TYPE_FIXED_SIZE_LIST:
2380
+ case NANOARROW_TYPE_STRUCT:
2381
+ case NANOARROW_TYPE_SPARSE_UNION:
2382
+ array->n_buffers = 1;
2383
+ break;
2384
+
2385
+ case NANOARROW_TYPE_LIST:
2386
+ case NANOARROW_TYPE_LARGE_LIST:
2387
+ case NANOARROW_TYPE_MAP:
2388
+ case NANOARROW_TYPE_BOOL:
2389
+ case NANOARROW_TYPE_UINT8:
2390
+ case NANOARROW_TYPE_INT8:
2391
+ case NANOARROW_TYPE_UINT16:
2392
+ case NANOARROW_TYPE_INT16:
2393
+ case NANOARROW_TYPE_UINT32:
2394
+ case NANOARROW_TYPE_INT32:
2395
+ case NANOARROW_TYPE_UINT64:
2396
+ case NANOARROW_TYPE_INT64:
2397
+ case NANOARROW_TYPE_HALF_FLOAT:
2398
+ case NANOARROW_TYPE_FLOAT:
2399
+ case NANOARROW_TYPE_DOUBLE:
2400
+ case NANOARROW_TYPE_DECIMAL32:
2401
+ case NANOARROW_TYPE_DECIMAL64:
2402
+ case NANOARROW_TYPE_DECIMAL128:
2403
+ case NANOARROW_TYPE_DECIMAL256:
2404
+ case NANOARROW_TYPE_INTERVAL_MONTHS:
2405
+ case NANOARROW_TYPE_INTERVAL_DAY_TIME:
2406
+ case NANOARROW_TYPE_INTERVAL_MONTH_DAY_NANO:
2407
+ case NANOARROW_TYPE_FIXED_SIZE_BINARY:
2408
+ case NANOARROW_TYPE_DENSE_UNION:
2409
+ array->n_buffers = 2;
2410
+ break;
2411
+ case NANOARROW_TYPE_BINARY_VIEW:
2412
+ case NANOARROW_TYPE_STRING_VIEW:
2413
+ array->n_buffers = NANOARROW_BINARY_VIEW_FIXED_BUFFERS + 1;
2414
+ break;
2415
+ case NANOARROW_TYPE_STRING:
2416
+ case NANOARROW_TYPE_LARGE_STRING:
2417
+ case NANOARROW_TYPE_BINARY:
2418
+ case NANOARROW_TYPE_LARGE_BINARY:
2419
+ case NANOARROW_TYPE_LIST_VIEW:
2420
+ case NANOARROW_TYPE_LARGE_LIST_VIEW:
2421
+ array->n_buffers = 3;
2422
+ break;
2423
+
2424
+ default:
2425
+ return EINVAL;
2426
+
2427
+ return NANOARROW_OK;
2428
+ }
2429
+
2430
+ struct ArrowArrayPrivateData* private_data =
2431
+ (struct ArrowArrayPrivateData*)array->private_data;
2432
+ private_data->storage_type = storage_type;
2433
+ return NANOARROW_OK;
2434
+ }
2435
+
2436
+ ArrowErrorCode ArrowArrayInitFromType(struct ArrowArray* array,
2437
+ enum ArrowType storage_type) {
2438
+ array->length = 0;
2439
+ array->null_count = 0;
2440
+ array->offset = 0;
2441
+ array->n_buffers = 0;
2442
+ array->n_children = 0;
2443
+ array->buffers = NULL;
2444
+ array->children = NULL;
2445
+ array->dictionary = NULL;
2446
+ array->release = &ArrowArrayReleaseInternal;
2447
+ array->private_data = NULL;
2448
+
2449
+ struct ArrowArrayPrivateData* private_data =
2450
+ (struct ArrowArrayPrivateData*)ArrowMalloc(sizeof(struct ArrowArrayPrivateData));
2451
+ if (private_data == NULL) {
2452
+ array->release = NULL;
2453
+ return ENOMEM;
2454
+ }
2455
+
2456
+ ArrowBitmapInit(&private_data->bitmap);
2457
+ ArrowBufferInit(&private_data->buffers[0]);
2458
+ ArrowBufferInit(&private_data->buffers[1]);
2459
+ private_data->buffer_data =
2460
+ (const void**)ArrowMalloc(sizeof(void*) * NANOARROW_MAX_FIXED_BUFFERS);
2461
+ for (int i = 0; i < NANOARROW_MAX_FIXED_BUFFERS; ++i) {
2462
+ private_data->buffer_data[i] = NULL;
2463
+ }
2464
+ private_data->n_variadic_buffers = 0;
2465
+ private_data->variadic_buffers = NULL;
2466
+ private_data->list_view_offset = 0;
2467
+
2468
+ array->private_data = private_data;
2469
+ array->buffers = (const void**)(private_data->buffer_data);
2470
+
2471
+ // These are not technically "storage" in the sense that they do not appear
2472
+ // in the ArrowSchemaView's storage_type member; however, allowing them here
2473
+ // is helpful to maximize the number of types that can avoid going through
2474
+ // ArrowArrayInitFromSchema().
2475
+ switch (storage_type) {
2476
+ case NANOARROW_TYPE_DURATION:
2477
+ case NANOARROW_TYPE_TIMESTAMP:
2478
+ case NANOARROW_TYPE_TIME64:
2479
+ case NANOARROW_TYPE_DATE64:
2480
+ storage_type = NANOARROW_TYPE_INT64;
2481
+ break;
2482
+ case NANOARROW_TYPE_TIME32:
2483
+ case NANOARROW_TYPE_DATE32:
2484
+ storage_type = NANOARROW_TYPE_INT32;
2485
+ break;
2486
+ default:
2487
+ break;
2488
+ }
2489
+
2490
+ int result = ArrowArraySetStorageType(array, storage_type);
2491
+ if (result != NANOARROW_OK) {
2492
+ ArrowArrayRelease(array);
2493
+ return result;
2494
+ }
2495
+
2496
+ ArrowLayoutInit(&private_data->layout, storage_type);
2497
+ // We can only know this not to be true when initializing based on a schema
2498
+ // so assume this to be true.
2499
+ private_data->union_type_id_is_child_index = 1;
2500
+ return NANOARROW_OK;
2501
+ }
2502
+
2503
+ ArrowErrorCode ArrowArrayInitFromArrayView(struct ArrowArray* array,
2504
+ const struct ArrowArrayView* array_view,
2505
+ struct ArrowError* error) {
2506
+ NANOARROW_RETURN_NOT_OK_WITH_ERROR(
2507
+ ArrowArrayInitFromType(array, array_view->storage_type), error);
2508
+ int result;
2509
+
2510
+ struct ArrowArrayPrivateData* private_data =
2511
+ (struct ArrowArrayPrivateData*)array->private_data;
2512
+ private_data->layout = array_view->layout;
2513
+
2514
+ if (array_view->n_children > 0) {
2515
+ result = ArrowArrayAllocateChildren(array, array_view->n_children);
2516
+ if (result != NANOARROW_OK) {
2517
+ ArrowArrayRelease(array);
2518
+ return result;
2519
+ }
2520
+
2521
+ for (int64_t i = 0; i < array_view->n_children; i++) {
2522
+ result =
2523
+ ArrowArrayInitFromArrayView(array->children[i], array_view->children[i], error);
2524
+ if (result != NANOARROW_OK) {
2525
+ ArrowArrayRelease(array);
2526
+ return result;
2527
+ }
2528
+ }
2529
+ }
2530
+
2531
+ if (array_view->dictionary != NULL) {
2532
+ result = ArrowArrayAllocateDictionary(array);
2533
+ if (result != NANOARROW_OK) {
2534
+ ArrowArrayRelease(array);
2535
+ return result;
2536
+ }
2537
+
2538
+ result =
2539
+ ArrowArrayInitFromArrayView(array->dictionary, array_view->dictionary, error);
2540
+ if (result != NANOARROW_OK) {
2541
+ ArrowArrayRelease(array);
2542
+ return result;
2543
+ }
2544
+ }
2545
+
2546
+ return NANOARROW_OK;
2547
+ }
2548
+
2549
+ ArrowErrorCode ArrowArrayInitFromSchema(struct ArrowArray* array,
2550
+ const struct ArrowSchema* schema,
2551
+ struct ArrowError* error) {
2552
+ struct ArrowArrayView array_view;
2553
+ NANOARROW_RETURN_NOT_OK(ArrowArrayViewInitFromSchema(&array_view, schema, error));
2554
+ NANOARROW_RETURN_NOT_OK(ArrowArrayInitFromArrayView(array, &array_view, error));
2555
+ if (array_view.storage_type == NANOARROW_TYPE_DENSE_UNION ||
2556
+ array_view.storage_type == NANOARROW_TYPE_SPARSE_UNION) {
2557
+ struct ArrowArrayPrivateData* private_data =
2558
+ (struct ArrowArrayPrivateData*)array->private_data;
2559
+ // We can still build arrays if this isn't true; however, the append
2560
+ // functions won't work. Instead, we store this value and error only
2561
+ // when StartAppending is called.
2562
+ private_data->union_type_id_is_child_index =
2563
+ _ArrowUnionTypeIdsWillEqualChildIndices(schema->format + 4, schema->n_children);
2564
+ }
2565
+
2566
+ ArrowArrayViewReset(&array_view);
2567
+ return NANOARROW_OK;
2568
+ }
2569
+
2570
+ ArrowErrorCode ArrowArrayAllocateChildren(struct ArrowArray* array, int64_t n_children) {
2571
+ if (array->children != NULL) {
2572
+ return EINVAL;
2573
+ }
2574
+
2575
+ if (n_children == 0) {
2576
+ return NANOARROW_OK;
2577
+ }
2578
+
2579
+ array->children =
2580
+ (struct ArrowArray**)ArrowMalloc(n_children * sizeof(struct ArrowArray*));
2581
+ if (array->children == NULL) {
2582
+ return ENOMEM;
2583
+ }
2584
+
2585
+ memset(array->children, 0, n_children * sizeof(struct ArrowArray*));
2586
+
2587
+ for (int64_t i = 0; i < n_children; i++) {
2588
+ array->children[i] = (struct ArrowArray*)ArrowMalloc(sizeof(struct ArrowArray));
2589
+ if (array->children[i] == NULL) {
2590
+ return ENOMEM;
2591
+ }
2592
+ array->children[i]->release = NULL;
2593
+ }
2594
+
2595
+ array->n_children = n_children;
2596
+ return NANOARROW_OK;
2597
+ }
2598
+
2599
+ ArrowErrorCode ArrowArrayAllocateDictionary(struct ArrowArray* array) {
2600
+ if (array->dictionary != NULL) {
2601
+ return EINVAL;
2602
+ }
2603
+
2604
+ array->dictionary = (struct ArrowArray*)ArrowMalloc(sizeof(struct ArrowArray));
2605
+ if (array->dictionary == NULL) {
2606
+ return ENOMEM;
2607
+ }
2608
+
2609
+ array->dictionary->release = NULL;
2610
+ return NANOARROW_OK;
2611
+ }
2612
+
2613
+ void ArrowArraySetValidityBitmap(struct ArrowArray* array, struct ArrowBitmap* bitmap) {
2614
+ struct ArrowArrayPrivateData* private_data =
2615
+ (struct ArrowArrayPrivateData*)array->private_data;
2616
+ ArrowBufferMove(&bitmap->buffer, &private_data->bitmap.buffer);
2617
+ private_data->bitmap.size_bits = bitmap->size_bits;
2618
+ bitmap->size_bits = 0;
2619
+ private_data->buffer_data[0] = private_data->bitmap.buffer.data;
2620
+ array->null_count = -1;
2621
+ }
2622
+
2623
+ ArrowErrorCode ArrowArraySetBuffer(struct ArrowArray* array, int64_t i,
2624
+ struct ArrowBuffer* buffer) {
2625
+ struct ArrowArrayPrivateData* private_data =
2626
+ (struct ArrowArrayPrivateData*)array->private_data;
2627
+
2628
+ if (i >= array->n_buffers || i < 0) {
2629
+ return EINVAL;
2630
+ }
2631
+
2632
+ // Find the `i`th buffer, release what is currently there, and move the
2633
+ // supplied buffer into that slot.
2634
+ struct ArrowBuffer* dst = ArrowArrayBuffer(array, i);
2635
+ ArrowBufferReset(dst);
2636
+ ArrowBufferMove(buffer, dst);
2637
+
2638
+ // Flush the pointer into array->buffers. In theory clients should call
2639
+ // ArrowArrayFinishBuilding() to flush the pointer values before passing
2640
+ // this array elsewhere; however, in early nanoarrow versions this was not
2641
+ // needed and some code may depend on this being true.
2642
+ private_data->buffer_data[i] = dst->data;
2643
+ array->buffers = private_data->buffer_data;
2644
+
2645
+ return NANOARROW_OK;
2646
+ }
2647
+
2648
+ static ArrowErrorCode ArrowArrayViewInitFromArray(struct ArrowArrayView* array_view,
2649
+ struct ArrowArray* array,
2650
+ struct ArrowError* error) {
2651
+ if (!ArrowArrayIsInternal(array)) {
2652
+ ArrowErrorSet(error,
2653
+ "Can't initialize internal ArrowArrayView from external ArrowArray");
2654
+ return EINVAL;
2655
+ }
2656
+
2657
+ struct ArrowArrayPrivateData* private_data =
2658
+ (struct ArrowArrayPrivateData*)array->private_data;
2659
+
2660
+ ArrowArrayViewInitFromType(array_view, private_data->storage_type);
2661
+ array_view->layout = private_data->layout;
2662
+ array_view->array = array;
2663
+ array_view->length = array->length;
2664
+ array_view->offset = array->offset;
2665
+ array_view->null_count = array->null_count;
2666
+
2667
+ array_view->buffer_views[0].data.as_uint8 = private_data->bitmap.buffer.data;
2668
+ array_view->buffer_views[0].size_bytes = private_data->bitmap.buffer.size_bytes;
2669
+ array_view->buffer_views[1].data.as_uint8 = private_data->buffers[0].data;
2670
+ array_view->buffer_views[1].size_bytes = private_data->buffers[0].size_bytes;
2671
+ array_view->buffer_views[2].data.as_uint8 = private_data->buffers[1].data;
2672
+ array_view->buffer_views[2].size_bytes = private_data->buffers[1].size_bytes;
2673
+
2674
+ int result = ArrowArrayViewAllocateChildren(array_view, array->n_children);
2675
+ if (result != NANOARROW_OK) {
2676
+ ArrowArrayViewReset(array_view);
2677
+ return result;
2678
+ }
2679
+
2680
+ for (int64_t i = 0; i < array->n_children; i++) {
2681
+ result =
2682
+ ArrowArrayViewInitFromArray(array_view->children[i], array->children[i], error);
2683
+ if (result != NANOARROW_OK) {
2684
+ ArrowArrayViewReset(array_view);
2685
+ return result;
2686
+ }
2687
+ }
2688
+
2689
+ if (array->dictionary != NULL) {
2690
+ result = ArrowArrayViewAllocateDictionary(array_view);
2691
+ if (result != NANOARROW_OK) {
2692
+ ArrowArrayViewReset(array_view);
2693
+ return result;
2694
+ }
2695
+
2696
+ result =
2697
+ ArrowArrayViewInitFromArray(array_view->dictionary, array->dictionary, error);
2698
+ if (result != NANOARROW_OK) {
2699
+ ArrowArrayViewReset(array_view);
2700
+ return result;
2701
+ }
2702
+ }
2703
+
2704
+ return NANOARROW_OK;
2705
+ }
2706
+
2707
+ static ArrowErrorCode ArrowArrayReserveInternal(struct ArrowArray* array,
2708
+ struct ArrowArrayView* array_view) {
2709
+ // Loop through buffers and reserve the extra space that we know about
2710
+ for (int64_t i = 0; i < NANOARROW_MAX_FIXED_BUFFERS; i++) {
2711
+ // Don't reserve on a validity buffer that hasn't been allocated yet
2712
+ if (array_view->layout.buffer_type[i] == NANOARROW_BUFFER_TYPE_VALIDITY &&
2713
+ ArrowArrayBuffer(array, i)->data == NULL) {
2714
+ continue;
2715
+ }
2716
+
2717
+ int64_t additional_size_bytes =
2718
+ array_view->buffer_views[i].size_bytes - ArrowArrayBuffer(array, i)->size_bytes;
2719
+
2720
+ if (additional_size_bytes > 0) {
2721
+ NANOARROW_RETURN_NOT_OK(
2722
+ ArrowBufferReserve(ArrowArrayBuffer(array, i), additional_size_bytes));
2723
+ }
2724
+ }
2725
+
2726
+ // Recursively reserve children
2727
+ for (int64_t i = 0; i < array->n_children; i++) {
2728
+ NANOARROW_RETURN_NOT_OK(
2729
+ ArrowArrayReserveInternal(array->children[i], array_view->children[i]));
2730
+ }
2731
+
2732
+ return NANOARROW_OK;
2733
+ }
2734
+
2735
+ ArrowErrorCode ArrowArrayReserve(struct ArrowArray* array,
2736
+ int64_t additional_size_elements) {
2737
+ struct ArrowArrayView array_view;
2738
+ NANOARROW_RETURN_NOT_OK(ArrowArrayViewInitFromArray(&array_view, array, NULL));
2739
+
2740
+ // Calculate theoretical buffer sizes (recursively)
2741
+ ArrowArrayViewSetLength(&array_view, array->length + additional_size_elements);
2742
+
2743
+ // Walk the structure (recursively)
2744
+ int result = ArrowArrayReserveInternal(array, &array_view);
2745
+ ArrowArrayViewReset(&array_view);
2746
+ if (result != NANOARROW_OK) {
2747
+ return result;
2748
+ }
2749
+
2750
+ return NANOARROW_OK;
2751
+ }
2752
+
2753
+ static ArrowErrorCode ArrowArrayFinalizeBuffers(struct ArrowArray* array) {
2754
+ struct ArrowArrayPrivateData* private_data =
2755
+ (struct ArrowArrayPrivateData*)array->private_data;
2756
+
2757
+ for (int i = 0; i < NANOARROW_MAX_FIXED_BUFFERS; i++) {
2758
+ if (private_data->layout.buffer_type[i] == NANOARROW_BUFFER_TYPE_VALIDITY ||
2759
+ private_data->layout.buffer_type[i] == NANOARROW_BUFFER_TYPE_NONE) {
2760
+ continue;
2761
+ }
2762
+
2763
+ struct ArrowBuffer* buffer = ArrowArrayBuffer(array, i);
2764
+ if (buffer->data == NULL) {
2765
+ NANOARROW_RETURN_NOT_OK((ArrowBufferReserve(buffer, 1)));
2766
+ }
2767
+ }
2768
+
2769
+ for (int64_t i = 0; i < array->n_children; i++) {
2770
+ if (ArrowArrayIsInternal(array->children[i])) {
2771
+ NANOARROW_RETURN_NOT_OK(ArrowArrayFinalizeBuffers(array->children[i]));
2772
+ }
2773
+ }
2774
+
2775
+ if (array->dictionary != NULL && ArrowArrayIsInternal(array->dictionary)) {
2776
+ NANOARROW_RETURN_NOT_OK(ArrowArrayFinalizeBuffers(array->dictionary));
2777
+ }
2778
+
2779
+ return NANOARROW_OK;
2780
+ }
2781
+
2782
+ static ArrowErrorCode ArrowArrayFlushInternalPointers(struct ArrowArray* array) {
2783
+ NANOARROW_DCHECK(ArrowArrayIsInternal(array));
2784
+ struct ArrowArrayPrivateData* private_data =
2785
+ (struct ArrowArrayPrivateData*)array->private_data;
2786
+
2787
+ if (array->n_buffers > NANOARROW_MAX_FIXED_BUFFERS) {
2788
+ // If the variadic sizes buffer was not set and there is at least one variadic
2789
+ // buffer, populate it now (if there are no variadic buffers there will be exactly
2790
+ // three total buffers and we don't need to do anything special here). Notably, this
2791
+ // will occur when building a BinaryView/StringView array by element using the
2792
+ // appender.
2793
+ struct ArrowBuffer* sizes_buffer = ArrowArrayBuffer(array, array->n_buffers - 1);
2794
+ if (sizes_buffer->data == NULL && sizes_buffer->size_bytes == 0) {
2795
+ NANOARROW_RETURN_NOT_OK(
2796
+ ArrowBufferReserve(sizes_buffer, private_data->n_variadic_buffers));
2797
+ for (int64_t i = 0; i < private_data->n_variadic_buffers; i++) {
2798
+ struct ArrowBuffer* variadic_buffer =
2799
+ ArrowArrayBuffer(array, i + NANOARROW_BINARY_VIEW_FIXED_BUFFERS);
2800
+ NANOARROW_RETURN_NOT_OK(
2801
+ ArrowBufferAppendInt64(sizes_buffer, variadic_buffer->size_bytes));
2802
+ }
2803
+ }
2804
+ }
2805
+
2806
+ for (int32_t i = 0; i < array->n_buffers; i++) {
2807
+ private_data->buffer_data[i] = ArrowArrayBuffer(array, i)->data;
2808
+ }
2809
+
2810
+ array->buffers = (const void**)(private_data->buffer_data);
2811
+
2812
+ // Flush internal pointers for child/dictionary arrays if we allocated them. Clients
2813
+ // building arrays by buffer might have moved arrays from some other source (e.g.,
2814
+ // to create a record batch) and calling this function in that case will cause a crash.
2815
+ for (int64_t i = 0; i < array->n_children; i++) {
2816
+ if (ArrowArrayIsInternal(array->children[i])) {
2817
+ NANOARROW_RETURN_NOT_OK(ArrowArrayFlushInternalPointers(array->children[i]));
2818
+ }
2819
+ }
2820
+
2821
+ if (array->dictionary != NULL && ArrowArrayIsInternal(array->dictionary)) {
2822
+ NANOARROW_RETURN_NOT_OK(ArrowArrayFlushInternalPointers(array->dictionary));
2823
+ }
2824
+
2825
+ return NANOARROW_OK;
2826
+ }
2827
+
2828
+ ArrowErrorCode ArrowArrayFinishBuilding(struct ArrowArray* array,
2829
+ enum ArrowValidationLevel validation_level,
2830
+ struct ArrowError* error) {
2831
+ // Even if the data buffer is size zero, the pointer value needed to be non-null
2832
+ // in some implementations (at least one version of Arrow C++ at the time this
2833
+ // was added and C# as later discovered). Only do this fix if we can assume
2834
+ // CPU data access.
2835
+ if (validation_level >= NANOARROW_VALIDATION_LEVEL_DEFAULT) {
2836
+ NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowArrayFinalizeBuffers(array), error);
2837
+ }
2838
+
2839
+ // Make sure the value we get with array->buffers[i] is set to the actual
2840
+ // pointer (which may have changed from the original due to reallocation)
2841
+ NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowArrayFlushInternalPointers(array), error);
2842
+
2843
+ if (validation_level == NANOARROW_VALIDATION_LEVEL_NONE) {
2844
+ return NANOARROW_OK;
2845
+ }
2846
+
2847
+ // For validation, initialize an ArrowArrayView with our known buffer sizes
2848
+ struct ArrowArrayView array_view;
2849
+ NANOARROW_RETURN_NOT_OK_WITH_ERROR(
2850
+ ArrowArrayViewInitFromArray(&array_view, array, error), error);
2851
+ int result = ArrowArrayViewValidate(&array_view, validation_level, error);
2852
+ ArrowArrayViewReset(&array_view);
2853
+ return result;
2854
+ }
2855
+
2856
+ ArrowErrorCode ArrowArrayFinishBuildingDefault(struct ArrowArray* array,
2857
+ struct ArrowError* error) {
2858
+ return ArrowArrayFinishBuilding(array, NANOARROW_VALIDATION_LEVEL_DEFAULT, error);
2859
+ }
2860
+
2861
+ void ArrowArrayViewInitFromType(struct ArrowArrayView* array_view,
2862
+ enum ArrowType storage_type) {
2863
+ memset(array_view, 0, sizeof(struct ArrowArrayView));
2864
+ array_view->storage_type = storage_type;
2865
+ ArrowLayoutInit(&array_view->layout, storage_type);
2866
+ }
2867
+
2868
+ ArrowErrorCode ArrowArrayViewAllocateChildren(struct ArrowArrayView* array_view,
2869
+ int64_t n_children) {
2870
+ if (array_view->children != NULL) {
2871
+ return EINVAL;
2872
+ }
2873
+
2874
+ if (n_children == 0) {
2875
+ array_view->n_children = 0;
2876
+ return NANOARROW_OK;
2877
+ }
2878
+
2879
+ array_view->children =
2880
+ (struct ArrowArrayView**)ArrowMalloc(n_children * sizeof(struct ArrowArrayView*));
2881
+ if (array_view->children == NULL) {
2882
+ return ENOMEM;
2883
+ }
2884
+
2885
+ for (int64_t i = 0; i < n_children; i++) {
2886
+ array_view->children[i] = NULL;
2887
+ }
2888
+
2889
+ array_view->n_children = n_children;
2890
+
2891
+ for (int64_t i = 0; i < n_children; i++) {
2892
+ array_view->children[i] =
2893
+ (struct ArrowArrayView*)ArrowMalloc(sizeof(struct ArrowArrayView));
2894
+ if (array_view->children[i] == NULL) {
2895
+ return ENOMEM;
2896
+ }
2897
+ ArrowArrayViewInitFromType(array_view->children[i], NANOARROW_TYPE_UNINITIALIZED);
2898
+ }
2899
+
2900
+ return NANOARROW_OK;
2901
+ }
2902
+
2903
+ ArrowErrorCode ArrowArrayViewAllocateDictionary(struct ArrowArrayView* array_view) {
2904
+ if (array_view->dictionary != NULL) {
2905
+ return EINVAL;
2906
+ }
2907
+
2908
+ array_view->dictionary =
2909
+ (struct ArrowArrayView*)ArrowMalloc(sizeof(struct ArrowArrayView));
2910
+ if (array_view->dictionary == NULL) {
2911
+ return ENOMEM;
2912
+ }
2913
+
2914
+ ArrowArrayViewInitFromType(array_view->dictionary, NANOARROW_TYPE_UNINITIALIZED);
2915
+ return NANOARROW_OK;
2916
+ }
2917
+
2918
+ ArrowErrorCode ArrowArrayViewInitFromSchema(struct ArrowArrayView* array_view,
2919
+ const struct ArrowSchema* schema,
2920
+ struct ArrowError* error) {
2921
+ struct ArrowSchemaView schema_view;
2922
+ int result = ArrowSchemaViewInit(&schema_view, schema, error);
2923
+ if (result != NANOARROW_OK) {
2924
+ return result;
2925
+ }
2926
+
2927
+ ArrowArrayViewInitFromType(array_view, schema_view.storage_type);
2928
+ array_view->layout = schema_view.layout;
2929
+
2930
+ result = ArrowArrayViewAllocateChildren(array_view, schema->n_children);
2931
+ if (result != NANOARROW_OK) {
2932
+ ArrowErrorSet(error, "ArrowArrayViewAllocateChildren() failed");
2933
+ ArrowArrayViewReset(array_view);
2934
+ return result;
2935
+ }
2936
+
2937
+ for (int64_t i = 0; i < schema->n_children; i++) {
2938
+ result =
2939
+ ArrowArrayViewInitFromSchema(array_view->children[i], schema->children[i], error);
2940
+ if (result != NANOARROW_OK) {
2941
+ ArrowArrayViewReset(array_view);
2942
+ return result;
2943
+ }
2944
+ }
2945
+
2946
+ if (schema->dictionary != NULL) {
2947
+ result = ArrowArrayViewAllocateDictionary(array_view);
2948
+ if (result != NANOARROW_OK) {
2949
+ ArrowArrayViewReset(array_view);
2950
+ return result;
2951
+ }
2952
+
2953
+ result =
2954
+ ArrowArrayViewInitFromSchema(array_view->dictionary, schema->dictionary, error);
2955
+ if (result != NANOARROW_OK) {
2956
+ ArrowArrayViewReset(array_view);
2957
+ return result;
2958
+ }
2959
+ }
2960
+
2961
+ if (array_view->storage_type == NANOARROW_TYPE_SPARSE_UNION ||
2962
+ array_view->storage_type == NANOARROW_TYPE_DENSE_UNION) {
2963
+ array_view->union_type_id_map = (int8_t*)ArrowMalloc(256 * sizeof(int8_t));
2964
+ if (array_view->union_type_id_map == NULL) {
2965
+ return ENOMEM;
2966
+ }
2967
+
2968
+ memset(array_view->union_type_id_map, -1, 256);
2969
+ int32_t n_type_ids = _ArrowParseUnionTypeIds(schema_view.union_type_ids,
2970
+ array_view->union_type_id_map + 128);
2971
+ for (int8_t child_index = 0; child_index < n_type_ids; child_index++) {
2972
+ int8_t type_id = array_view->union_type_id_map[128 + child_index];
2973
+ array_view->union_type_id_map[type_id] = child_index;
2974
+ }
2975
+ }
2976
+
2977
+ return NANOARROW_OK;
2978
+ }
2979
+
2980
+ void ArrowArrayViewReset(struct ArrowArrayView* array_view) {
2981
+ if (array_view->children != NULL) {
2982
+ for (int64_t i = 0; i < array_view->n_children; i++) {
2983
+ if (array_view->children[i] != NULL) {
2984
+ ArrowArrayViewReset(array_view->children[i]);
2985
+ ArrowFree(array_view->children[i]);
2986
+ }
2987
+ }
2988
+
2989
+ ArrowFree(array_view->children);
2990
+ }
2991
+
2992
+ if (array_view->dictionary != NULL) {
2993
+ ArrowArrayViewReset(array_view->dictionary);
2994
+ ArrowFree(array_view->dictionary);
2995
+ }
2996
+
2997
+ if (array_view->union_type_id_map != NULL) {
2998
+ ArrowFree(array_view->union_type_id_map);
2999
+ }
3000
+
3001
+ ArrowArrayViewInitFromType(array_view, NANOARROW_TYPE_UNINITIALIZED);
3002
+ }
3003
+
3004
+ void ArrowArrayViewSetLength(struct ArrowArrayView* array_view, int64_t length) {
3005
+ for (int i = 0; i < NANOARROW_MAX_FIXED_BUFFERS; i++) {
3006
+ int64_t element_size_bytes = array_view->layout.element_size_bits[i] / 8;
3007
+
3008
+ switch (array_view->layout.buffer_type[i]) {
3009
+ case NANOARROW_BUFFER_TYPE_VALIDITY:
3010
+ array_view->buffer_views[i].size_bytes = _ArrowBytesForBits(length);
3011
+ continue;
3012
+ case NANOARROW_BUFFER_TYPE_DATA_OFFSET:
3013
+ // Probably don't want/need to rely on the producer to have allocated an
3014
+ // offsets buffer of length 1 for a zero-size array
3015
+ array_view->buffer_views[i].size_bytes =
3016
+ (length != 0) * element_size_bytes * (length + 1);
3017
+ continue;
3018
+ case NANOARROW_BUFFER_TYPE_DATA:
3019
+ array_view->buffer_views[i].size_bytes =
3020
+ _ArrowRoundUpToMultipleOf8(array_view->layout.element_size_bits[i] * length) /
3021
+ 8;
3022
+ continue;
3023
+ case NANOARROW_BUFFER_TYPE_TYPE_ID:
3024
+ case NANOARROW_BUFFER_TYPE_UNION_OFFSET:
3025
+ case NANOARROW_BUFFER_TYPE_VIEW_OFFSET:
3026
+ case NANOARROW_BUFFER_TYPE_SIZE:
3027
+ array_view->buffer_views[i].size_bytes = element_size_bytes * length;
3028
+ continue;
3029
+ case NANOARROW_BUFFER_TYPE_VARIADIC_DATA:
3030
+ case NANOARROW_BUFFER_TYPE_VARIADIC_SIZE:
3031
+ case NANOARROW_BUFFER_TYPE_NONE:
3032
+ array_view->buffer_views[i].size_bytes = 0;
3033
+ continue;
3034
+ }
3035
+ }
3036
+
3037
+ switch (array_view->storage_type) {
3038
+ case NANOARROW_TYPE_STRUCT:
3039
+ case NANOARROW_TYPE_SPARSE_UNION:
3040
+ for (int64_t i = 0; i < array_view->n_children; i++) {
3041
+ ArrowArrayViewSetLength(array_view->children[i], length);
3042
+ }
3043
+ break;
3044
+ case NANOARROW_TYPE_FIXED_SIZE_LIST:
3045
+ if (array_view->n_children >= 1) {
3046
+ ArrowArrayViewSetLength(array_view->children[0],
3047
+ length * array_view->layout.child_size_elements);
3048
+ }
3049
+ default:
3050
+ break;
3051
+ }
3052
+ }
3053
+
3054
+ // This version recursively extracts information from the array and stores it
3055
+ // in the array view, performing any checks that require the original array.
3056
+ static int ArrowArrayViewSetArrayInternal(struct ArrowArrayView* array_view,
3057
+ const struct ArrowArray* array,
3058
+ struct ArrowError* error) {
3059
+ array_view->array = array;
3060
+ array_view->offset = array->offset;
3061
+ array_view->length = array->length;
3062
+ array_view->null_count = array->null_count;
3063
+ array_view->variadic_buffer_sizes = NULL;
3064
+ array_view->variadic_buffers = NULL;
3065
+ array_view->n_variadic_buffers = 0;
3066
+
3067
+ int64_t buffers_required = 0;
3068
+ const int nfixed_buf = array_view->storage_type == NANOARROW_TYPE_STRING_VIEW ||
3069
+ array_view->storage_type == NANOARROW_TYPE_BINARY_VIEW
3070
+ ? NANOARROW_BINARY_VIEW_FIXED_BUFFERS
3071
+ : NANOARROW_MAX_FIXED_BUFFERS;
3072
+ for (int i = 0; i < nfixed_buf; i++) {
3073
+ if (array_view->layout.buffer_type[i] == NANOARROW_BUFFER_TYPE_NONE) {
3074
+ break;
3075
+ }
3076
+
3077
+ buffers_required++;
3078
+
3079
+ // Set buffer pointer
3080
+ array_view->buffer_views[i].data.data = array->buffers[i];
3081
+
3082
+ // If non-null, set buffer size to unknown.
3083
+ if (array->buffers[i] == NULL) {
3084
+ array_view->buffer_views[i].size_bytes = 0;
3085
+ } else {
3086
+ array_view->buffer_views[i].size_bytes = -1;
3087
+ }
3088
+ }
3089
+
3090
+ if (array_view->storage_type == NANOARROW_TYPE_STRING_VIEW ||
3091
+ array_view->storage_type == NANOARROW_TYPE_BINARY_VIEW) {
3092
+ const int64_t n_buffers = array->n_buffers;
3093
+ const int32_t nfixed_buf = NANOARROW_BINARY_VIEW_FIXED_BUFFERS;
3094
+
3095
+ const int32_t nvariadic_buf = (int32_t)(n_buffers - nfixed_buf - 1);
3096
+ array_view->n_variadic_buffers = nvariadic_buf;
3097
+ buffers_required += nvariadic_buf + 1;
3098
+ array_view->variadic_buffers = array->buffers + NANOARROW_BINARY_VIEW_FIXED_BUFFERS;
3099
+ array_view->variadic_buffer_sizes = (int64_t*)array->buffers[n_buffers - 1];
3100
+ }
3101
+
3102
+ if (buffers_required != array->n_buffers) {
3103
+ ArrowErrorSet(error,
3104
+ "Expected array with %" PRId64 " buffer(s) but found %" PRId64
3105
+ " buffer(s)",
3106
+ buffers_required, array->n_buffers);
3107
+ return EINVAL;
3108
+ }
3109
+
3110
+ // Check number of children
3111
+ if (array_view->n_children != array->n_children) {
3112
+ ArrowErrorSet(error, "Expected %" PRId64 " children but found %" PRId64 " children",
3113
+ array_view->n_children, array->n_children);
3114
+ return EINVAL;
3115
+ }
3116
+
3117
+ // Recurse for children
3118
+ for (int64_t i = 0; i < array_view->n_children; i++) {
3119
+ NANOARROW_RETURN_NOT_OK(ArrowArrayViewSetArrayInternal(array_view->children[i],
3120
+ array->children[i], error));
3121
+ }
3122
+
3123
+ // Check dictionary
3124
+ if (array->dictionary == NULL && array_view->dictionary != NULL) {
3125
+ ArrowErrorSet(error, "Expected dictionary but found NULL");
3126
+ return EINVAL;
3127
+ }
3128
+
3129
+ if (array->dictionary != NULL && array_view->dictionary == NULL) {
3130
+ ArrowErrorSet(error, "Expected NULL dictionary but found dictionary member");
3131
+ return EINVAL;
3132
+ }
3133
+
3134
+ if (array->dictionary != NULL) {
3135
+ NANOARROW_RETURN_NOT_OK(
3136
+ ArrowArrayViewSetArrayInternal(array_view->dictionary, array->dictionary, error));
3137
+ }
3138
+
3139
+ return NANOARROW_OK;
3140
+ }
3141
+
3142
+ static int ArrowArrayViewValidateMinimal(struct ArrowArrayView* array_view,
3143
+ struct ArrowError* error) {
3144
+ if (array_view->length < 0) {
3145
+ ArrowErrorSet(error, "Expected length >= 0 but found length %" PRId64,
3146
+ array_view->length);
3147
+ return EINVAL;
3148
+ }
3149
+
3150
+ if (array_view->offset < 0) {
3151
+ ArrowErrorSet(error, "Expected offset >= 0 but found offset %" PRId64,
3152
+ array_view->offset);
3153
+ return EINVAL;
3154
+ }
3155
+
3156
+ // Ensure that offset + length fits within an int64 before a possible overflow
3157
+ if ((uint64_t)array_view->offset + (uint64_t)array_view->length > (uint64_t)INT64_MAX) {
3158
+ ArrowErrorSet(error, "Offset + length is > INT64_MAX");
3159
+ return EINVAL;
3160
+ }
3161
+
3162
+ // Calculate buffer sizes that do not require buffer access. If marked as
3163
+ // unknown, assign the buffer size; otherwise, validate it.
3164
+ int64_t offset_plus_length = array_view->offset + array_view->length;
3165
+
3166
+ // Only loop over the first two buffers because the size of the third buffer
3167
+ // is always data dependent for all current Arrow types.
3168
+ for (int i = 0; i < 2; i++) {
3169
+ int64_t element_size_bytes = array_view->layout.element_size_bits[i] / 8;
3170
+ // Initialize with a value that will cause an error if accidentally used uninitialized
3171
+ // Need to suppress the clang-tidy warning because gcc warns for possible use
3172
+ int64_t min_buffer_size_bytes = // NOLINT(clang-analyzer-deadcode.DeadStores)
3173
+ array_view->buffer_views[i].size_bytes + 1;
3174
+
3175
+ switch (array_view->layout.buffer_type[i]) {
3176
+ case NANOARROW_BUFFER_TYPE_VALIDITY:
3177
+ if (array_view->null_count == 0 && array_view->buffer_views[i].size_bytes == 0) {
3178
+ continue;
3179
+ }
3180
+
3181
+ min_buffer_size_bytes = _ArrowBytesForBits(offset_plus_length);
3182
+ break;
3183
+ case NANOARROW_BUFFER_TYPE_SIZE:
3184
+ min_buffer_size_bytes = element_size_bytes * offset_plus_length;
3185
+ break;
3186
+ case NANOARROW_BUFFER_TYPE_DATA_OFFSET:
3187
+ // Probably don't want/need to rely on the producer to have allocated an
3188
+ // offsets buffer of length 1 for a zero-size array
3189
+ min_buffer_size_bytes =
3190
+ (offset_plus_length != 0) * element_size_bytes * (offset_plus_length + 1);
3191
+ break;
3192
+ case NANOARROW_BUFFER_TYPE_VIEW_OFFSET:
3193
+ min_buffer_size_bytes =
3194
+ (offset_plus_length != 0) * element_size_bytes * offset_plus_length;
3195
+ break;
3196
+ case NANOARROW_BUFFER_TYPE_DATA:
3197
+ min_buffer_size_bytes =
3198
+ _ArrowRoundUpToMultipleOf8(array_view->layout.element_size_bits[i] *
3199
+ offset_plus_length) /
3200
+ 8;
3201
+ break;
3202
+ case NANOARROW_BUFFER_TYPE_TYPE_ID:
3203
+ case NANOARROW_BUFFER_TYPE_UNION_OFFSET:
3204
+ min_buffer_size_bytes = element_size_bytes * offset_plus_length;
3205
+ break;
3206
+ case NANOARROW_BUFFER_TYPE_VARIADIC_DATA:
3207
+ case NANOARROW_BUFFER_TYPE_VARIADIC_SIZE:
3208
+ case NANOARROW_BUFFER_TYPE_NONE:
3209
+ continue;
3210
+ }
3211
+
3212
+ // Assign or validate buffer size
3213
+ if (array_view->buffer_views[i].size_bytes == -1) {
3214
+ array_view->buffer_views[i].size_bytes = min_buffer_size_bytes;
3215
+ } else if (array_view->buffer_views[i].size_bytes < min_buffer_size_bytes) {
3216
+ ArrowErrorSet(error,
3217
+ "Expected %s array buffer %d to have size >= %" PRId64
3218
+ " bytes but found "
3219
+ "buffer with %" PRId64 " bytes",
3220
+ ArrowTypeString(array_view->storage_type), i, min_buffer_size_bytes,
3221
+ array_view->buffer_views[i].size_bytes);
3222
+ return EINVAL;
3223
+ }
3224
+ }
3225
+
3226
+ // For list, fixed-size list and map views, we can validate the number of children
3227
+ switch (array_view->storage_type) {
3228
+ case NANOARROW_TYPE_LIST:
3229
+ case NANOARROW_TYPE_LARGE_LIST:
3230
+ case NANOARROW_TYPE_FIXED_SIZE_LIST:
3231
+ case NANOARROW_TYPE_MAP:
3232
+ case NANOARROW_TYPE_LIST_VIEW:
3233
+ case NANOARROW_TYPE_LARGE_LIST_VIEW:
3234
+ if (array_view->n_children != 1) {
3235
+ ArrowErrorSet(error,
3236
+ "Expected 1 child of %s array but found %" PRId64 " child arrays",
3237
+ ArrowTypeString(array_view->storage_type), array_view->n_children);
3238
+ return EINVAL;
3239
+ }
3240
+ break;
3241
+ case NANOARROW_TYPE_RUN_END_ENCODED:
3242
+ if (array_view->n_children != 2) {
3243
+ ArrowErrorSet(
3244
+ error, "Expected 2 children for %s array but found %" PRId64 " child arrays",
3245
+ ArrowTypeString(array_view->storage_type), array_view->n_children);
3246
+ return EINVAL;
3247
+ }
3248
+ break;
3249
+ default:
3250
+ break;
3251
+ }
3252
+
3253
+ // For struct, the sparse union, and the fixed-size list views, we can validate child
3254
+ // lengths.
3255
+ int64_t child_min_length;
3256
+ switch (array_view->storage_type) {
3257
+ case NANOARROW_TYPE_SPARSE_UNION:
3258
+ case NANOARROW_TYPE_STRUCT:
3259
+ child_min_length = (array_view->offset + array_view->length);
3260
+ for (int64_t i = 0; i < array_view->n_children; i++) {
3261
+ if (array_view->children[i]->length < child_min_length) {
3262
+ ArrowErrorSet(error,
3263
+ "Expected struct child %" PRId64 " to have length >= %" PRId64
3264
+ " but found child with "
3265
+ "length %" PRId64,
3266
+ i + 1, child_min_length, array_view->children[i]->length);
3267
+ return EINVAL;
3268
+ }
3269
+ }
3270
+ break;
3271
+
3272
+ case NANOARROW_TYPE_FIXED_SIZE_LIST:
3273
+ child_min_length = (array_view->offset + array_view->length) *
3274
+ array_view->layout.child_size_elements;
3275
+ if (array_view->children[0]->length < child_min_length) {
3276
+ ArrowErrorSet(error,
3277
+ "Expected child of fixed_size_list array to have length >= %" PRId64
3278
+ " but "
3279
+ "found array with length %" PRId64,
3280
+ child_min_length, array_view->children[0]->length);
3281
+ return EINVAL;
3282
+ }
3283
+ break;
3284
+
3285
+ case NANOARROW_TYPE_RUN_END_ENCODED: {
3286
+ if (array_view->n_children != 2) {
3287
+ ArrowErrorSet(error,
3288
+ "Expected 2 children for run-end encoded array but found %" PRId64,
3289
+ array_view->n_children);
3290
+ return EINVAL;
3291
+ }
3292
+ struct ArrowArrayView* run_ends_view = array_view->children[0];
3293
+ struct ArrowArrayView* values_view = array_view->children[1];
3294
+ int64_t max_length;
3295
+ switch (run_ends_view->storage_type) {
3296
+ case NANOARROW_TYPE_INT16:
3297
+ max_length = INT16_MAX;
3298
+ break;
3299
+ case NANOARROW_TYPE_INT32:
3300
+ max_length = INT32_MAX;
3301
+ break;
3302
+ case NANOARROW_TYPE_INT64:
3303
+ max_length = INT64_MAX;
3304
+ break;
3305
+ default:
3306
+ ArrowErrorSet(
3307
+ error,
3308
+ "Run-end encoded array only supports INT16, INT32 or INT64 run-ends "
3309
+ "but found run-ends type %s",
3310
+ ArrowTypeString(run_ends_view->storage_type));
3311
+ return EINVAL;
3312
+ }
3313
+
3314
+ // There is already a check above that offset_plus_length < INT64_MAX
3315
+ if (offset_plus_length > max_length) {
3316
+ ArrowErrorSet(error,
3317
+ "Offset + length of a run-end encoded array must fit in a value"
3318
+ " of the run end type %s but is %" PRId64 " + %" PRId64,
3319
+ ArrowTypeString(run_ends_view->storage_type), array_view->offset,
3320
+ array_view->length);
3321
+ return EINVAL;
3322
+ }
3323
+
3324
+ if (run_ends_view->length > values_view->length) {
3325
+ ArrowErrorSet(error,
3326
+ "Length of run_ends is greater than the length of values: %" PRId64
3327
+ " > %" PRId64,
3328
+ run_ends_view->length, values_view->length);
3329
+ return EINVAL;
3330
+ }
3331
+
3332
+ if (run_ends_view->length == 0 && values_view->length != 0) {
3333
+ ArrowErrorSet(error,
3334
+ "Run-end encoded array has zero length %" PRId64
3335
+ ", but values array has "
3336
+ "non-zero length",
3337
+ values_view->length);
3338
+ return EINVAL;
3339
+ }
3340
+
3341
+ if (run_ends_view->null_count != 0) {
3342
+ ArrowErrorSet(error, "Null count must be 0 for run ends array, but is %" PRId64,
3343
+ run_ends_view->null_count);
3344
+ return EINVAL;
3345
+ }
3346
+ break;
3347
+ }
3348
+
3349
+ default:
3350
+ break;
3351
+ }
3352
+
3353
+ // Recurse for children
3354
+ for (int64_t i = 0; i < array_view->n_children; i++) {
3355
+ NANOARROW_RETURN_NOT_OK(
3356
+ ArrowArrayViewValidateMinimal(array_view->children[i], error));
3357
+ }
3358
+
3359
+ // Recurse for dictionary
3360
+ if (array_view->dictionary != NULL) {
3361
+ NANOARROW_RETURN_NOT_OK(ArrowArrayViewValidateMinimal(array_view->dictionary, error));
3362
+ }
3363
+
3364
+ return NANOARROW_OK;
3365
+ }
3366
+
3367
+ static int ArrowArrayViewValidateDefault(struct ArrowArrayView* array_view,
3368
+ struct ArrowError* error) {
3369
+ // Perform minimal validation. This will validate or assign
3370
+ // buffer sizes as long as buffer access is not required.
3371
+ NANOARROW_RETURN_NOT_OK(ArrowArrayViewValidateMinimal(array_view, error));
3372
+
3373
+ // Calculate buffer sizes or child lengths that require accessing the offsets
3374
+ // buffer. Where appropriate, validate that the first offset is >= 0.
3375
+ // If a buffer size is marked as unknown, assign it; otherwise, validate it.
3376
+ int64_t offset_plus_length = array_view->offset + array_view->length;
3377
+
3378
+ int64_t first_offset;
3379
+ int64_t last_offset;
3380
+ switch (array_view->storage_type) {
3381
+ case NANOARROW_TYPE_STRING:
3382
+ case NANOARROW_TYPE_BINARY:
3383
+ if (array_view->buffer_views[1].size_bytes != 0) {
3384
+ first_offset = array_view->buffer_views[1].data.as_int32[array_view->offset];
3385
+ if (first_offset < 0) {
3386
+ ArrowErrorSet(error, "Expected first offset >= 0 but found %" PRId64,
3387
+ first_offset);
3388
+ return EINVAL;
3389
+ }
3390
+
3391
+ last_offset = array_view->buffer_views[1].data.as_int32[offset_plus_length];
3392
+ if (last_offset < 0) {
3393
+ ArrowErrorSet(error, "Expected last offset >= 0 but found %" PRId64,
3394
+ last_offset);
3395
+ return EINVAL;
3396
+ }
3397
+
3398
+ // If the data buffer size is unknown, assign it; otherwise, check it
3399
+ if (array_view->buffer_views[2].size_bytes == -1) {
3400
+ array_view->buffer_views[2].size_bytes = last_offset;
3401
+ } else if (array_view->buffer_views[2].size_bytes < last_offset) {
3402
+ ArrowErrorSet(error,
3403
+ "Expected %s array buffer 2 to have size >= %" PRId64
3404
+ " bytes but found "
3405
+ "buffer with %" PRId64 " bytes",
3406
+ ArrowTypeString(array_view->storage_type), last_offset,
3407
+ array_view->buffer_views[2].size_bytes);
3408
+ return EINVAL;
3409
+ }
3410
+ } else if (array_view->buffer_views[2].size_bytes == -1) {
3411
+ // If the data buffer size is unknown and there are no bytes in the offset buffer,
3412
+ // set the data buffer size to 0.
3413
+ array_view->buffer_views[2].size_bytes = 0;
3414
+ }
3415
+ break;
3416
+
3417
+ case NANOARROW_TYPE_LARGE_STRING:
3418
+ case NANOARROW_TYPE_LARGE_BINARY:
3419
+ if (array_view->buffer_views[1].size_bytes != 0) {
3420
+ first_offset = array_view->buffer_views[1].data.as_int64[array_view->offset];
3421
+ if (first_offset < 0) {
3422
+ ArrowErrorSet(error, "Expected first offset >= 0 but found %" PRId64,
3423
+ first_offset);
3424
+ return EINVAL;
3425
+ }
3426
+
3427
+ last_offset = array_view->buffer_views[1].data.as_int64[offset_plus_length];
3428
+ if (last_offset < 0) {
3429
+ ArrowErrorSet(error, "Expected last offset >= 0 but found %" PRId64,
3430
+ last_offset);
3431
+ return EINVAL;
3432
+ }
3433
+
3434
+ // If the data buffer size is unknown, assign it; otherwise, check it
3435
+ if (array_view->buffer_views[2].size_bytes == -1) {
3436
+ array_view->buffer_views[2].size_bytes = last_offset;
3437
+ } else if (array_view->buffer_views[2].size_bytes < last_offset) {
3438
+ ArrowErrorSet(error,
3439
+ "Expected %s array buffer 2 to have size >= %" PRId64
3440
+ " bytes but found "
3441
+ "buffer with %" PRId64 " bytes",
3442
+ ArrowTypeString(array_view->storage_type), last_offset,
3443
+ array_view->buffer_views[2].size_bytes);
3444
+ return EINVAL;
3445
+ }
3446
+ } else if (array_view->buffer_views[2].size_bytes == -1) {
3447
+ // If the data buffer size is unknown and there are no bytes in the offset
3448
+ // buffer, set the data buffer size to 0.
3449
+ array_view->buffer_views[2].size_bytes = 0;
3450
+ }
3451
+ break;
3452
+
3453
+ case NANOARROW_TYPE_STRUCT:
3454
+ for (int64_t i = 0; i < array_view->n_children; i++) {
3455
+ if (array_view->children[i]->length < offset_plus_length) {
3456
+ ArrowErrorSet(error,
3457
+ "Expected struct child %" PRId64 " to have length >= %" PRId64
3458
+ " but found child with "
3459
+ "length %" PRId64,
3460
+ i + 1, offset_plus_length, array_view->children[i]->length);
3461
+ return EINVAL;
3462
+ }
3463
+ }
3464
+ break;
3465
+
3466
+ case NANOARROW_TYPE_LIST:
3467
+ case NANOARROW_TYPE_MAP:
3468
+ if (array_view->buffer_views[1].size_bytes != 0) {
3469
+ first_offset = array_view->buffer_views[1].data.as_int32[array_view->offset];
3470
+ if (first_offset < 0) {
3471
+ ArrowErrorSet(error, "Expected first offset >= 0 but found %" PRId64,
3472
+ first_offset);
3473
+ return EINVAL;
3474
+ }
3475
+
3476
+ last_offset = array_view->buffer_views[1].data.as_int32[offset_plus_length];
3477
+ if (last_offset < 0) {
3478
+ ArrowErrorSet(error, "Expected last offset >= 0 but found %" PRId64,
3479
+ last_offset);
3480
+ return EINVAL;
3481
+ }
3482
+
3483
+ if (array_view->children[0]->length < last_offset) {
3484
+ ArrowErrorSet(error,
3485
+ "Expected child of %s array to have length >= %" PRId64
3486
+ " but found array with "
3487
+ "length %" PRId64,
3488
+ ArrowTypeString(array_view->storage_type), last_offset,
3489
+ array_view->children[0]->length);
3490
+ return EINVAL;
3491
+ }
3492
+ }
3493
+ break;
3494
+
3495
+ case NANOARROW_TYPE_LARGE_LIST:
3496
+ if (array_view->buffer_views[1].size_bytes != 0) {
3497
+ first_offset = array_view->buffer_views[1].data.as_int64[array_view->offset];
3498
+ if (first_offset < 0) {
3499
+ ArrowErrorSet(error, "Expected first offset >= 0 but found %" PRId64,
3500
+ first_offset);
3501
+ return EINVAL;
3502
+ }
3503
+
3504
+ last_offset = array_view->buffer_views[1].data.as_int64[offset_plus_length];
3505
+ if (last_offset < 0) {
3506
+ ArrowErrorSet(error, "Expected last offset >= 0 but found %" PRId64,
3507
+ last_offset);
3508
+ return EINVAL;
3509
+ }
3510
+
3511
+ if (array_view->children[0]->length < last_offset) {
3512
+ ArrowErrorSet(error,
3513
+ "Expected child of %s array to have length >= %" PRId64
3514
+ " but found array "
3515
+ "with length %" PRId64,
3516
+ ArrowTypeString(array_view->storage_type), last_offset,
3517
+ array_view->children[0]->length);
3518
+ return EINVAL;
3519
+ }
3520
+ }
3521
+ break;
3522
+
3523
+ case NANOARROW_TYPE_RUN_END_ENCODED: {
3524
+ struct ArrowArrayView* run_ends_view = array_view->children[0];
3525
+ if (run_ends_view->length == 0) {
3526
+ break;
3527
+ }
3528
+
3529
+ int64_t first_run_end = ArrowArrayViewGetIntUnsafe(run_ends_view, 0);
3530
+ if (first_run_end < 1) {
3531
+ ArrowErrorSet(
3532
+ error,
3533
+ "All run ends must be greater than 0 but the first run end is %" PRId64,
3534
+ first_run_end);
3535
+ return EINVAL;
3536
+ }
3537
+
3538
+ // offset + length < INT64_MAX is checked in ArrowArrayViewValidateMinimal()
3539
+ int64_t last_run_end =
3540
+ ArrowArrayViewGetIntUnsafe(run_ends_view, run_ends_view->length - 1);
3541
+ if (last_run_end < offset_plus_length) {
3542
+ ArrowErrorSet(error,
3543
+ "Last run end is %" PRId64 " but it should be >= (%" PRId64
3544
+ " + %" PRId64 ")",
3545
+ last_run_end, array_view->offset, array_view->length);
3546
+ return EINVAL;
3547
+ }
3548
+ break;
3549
+ }
3550
+ default:
3551
+ break;
3552
+ }
3553
+
3554
+ // Recurse for children
3555
+ for (int64_t i = 0; i < array_view->n_children; i++) {
3556
+ NANOARROW_RETURN_NOT_OK(
3557
+ ArrowArrayViewValidateDefault(array_view->children[i], error));
3558
+ }
3559
+
3560
+ // Recurse for dictionary
3561
+ if (array_view->dictionary != NULL) {
3562
+ NANOARROW_RETURN_NOT_OK(ArrowArrayViewValidateDefault(array_view->dictionary, error));
3563
+ }
3564
+
3565
+ return NANOARROW_OK;
3566
+ }
3567
+
3568
+ ArrowErrorCode ArrowArrayViewSetArray(struct ArrowArrayView* array_view,
3569
+ const struct ArrowArray* array,
3570
+ struct ArrowError* error) {
3571
+ // Extract information from the array into the array view
3572
+ NANOARROW_RETURN_NOT_OK(ArrowArrayViewSetArrayInternal(array_view, array, error));
3573
+
3574
+ // Run default validation. Because we've marked all non-NULL buffers as having unknown
3575
+ // size, validation will also update the buffer sizes as it goes.
3576
+ NANOARROW_RETURN_NOT_OK(ArrowArrayViewValidateDefault(array_view, error));
3577
+
3578
+ return NANOARROW_OK;
3579
+ }
3580
+
3581
+ ArrowErrorCode ArrowArrayViewSetArrayMinimal(struct ArrowArrayView* array_view,
3582
+ const struct ArrowArray* array,
3583
+ struct ArrowError* error) {
3584
+ // Extract information from the array into the array view
3585
+ NANOARROW_RETURN_NOT_OK(ArrowArrayViewSetArrayInternal(array_view, array, error));
3586
+
3587
+ // Run default validation. Because we've marked all non-NULL buffers as having unknown
3588
+ // size, validation will also update the buffer sizes as it goes.
3589
+ NANOARROW_RETURN_NOT_OK(ArrowArrayViewValidateMinimal(array_view, error));
3590
+
3591
+ return NANOARROW_OK;
3592
+ }
3593
+
3594
+ static int ArrowAssertIncreasingInt32(struct ArrowBufferView view,
3595
+ struct ArrowError* error) {
3596
+ if (view.size_bytes <= (int64_t)sizeof(int32_t)) {
3597
+ return NANOARROW_OK;
3598
+ }
3599
+
3600
+ for (int64_t i = 1; i < view.size_bytes / (int64_t)sizeof(int32_t); i++) {
3601
+ if (view.data.as_int32[i] < view.data.as_int32[i - 1]) {
3602
+ ArrowErrorSet(error, "[%" PRId64 "] Expected element size >= 0", i);
3603
+ return EINVAL;
3604
+ }
3605
+ }
3606
+
3607
+ return NANOARROW_OK;
3608
+ }
3609
+
3610
+ static int ArrowAssertIncreasingInt64(struct ArrowBufferView view,
3611
+ struct ArrowError* error) {
3612
+ if (view.size_bytes <= (int64_t)sizeof(int64_t)) {
3613
+ return NANOARROW_OK;
3614
+ }
3615
+
3616
+ for (int64_t i = 1; i < view.size_bytes / (int64_t)sizeof(int64_t); i++) {
3617
+ if (view.data.as_int64[i] < view.data.as_int64[i - 1]) {
3618
+ ArrowErrorSet(error, "[%" PRId64 "] Expected element size >= 0", i);
3619
+ return EINVAL;
3620
+ }
3621
+ }
3622
+
3623
+ return NANOARROW_OK;
3624
+ }
3625
+
3626
+ static int ArrowAssertRangeInt8(struct ArrowBufferView view, int8_t min_value,
3627
+ int8_t max_value, struct ArrowError* error) {
3628
+ for (int64_t i = 0; i < view.size_bytes; i++) {
3629
+ if (view.data.as_int8[i] < min_value || view.data.as_int8[i] > max_value) {
3630
+ ArrowErrorSet(error,
3631
+ "[%" PRId64 "] Expected buffer value between %" PRId8 " and %" PRId8
3632
+ " but found value %" PRId8,
3633
+ i, min_value, max_value, view.data.as_int8[i]);
3634
+ return EINVAL;
3635
+ }
3636
+ }
3637
+
3638
+ return NANOARROW_OK;
3639
+ }
3640
+
3641
+ static int ArrowAssertInt8In(struct ArrowBufferView view, const int8_t* values,
3642
+ int64_t n_values, struct ArrowError* error) {
3643
+ for (int64_t i = 0; i < view.size_bytes; i++) {
3644
+ int item_found = 0;
3645
+ for (int64_t j = 0; j < n_values; j++) {
3646
+ if (view.data.as_int8[i] == values[j]) {
3647
+ item_found = 1;
3648
+ break;
3649
+ }
3650
+ }
3651
+
3652
+ if (!item_found) {
3653
+ ArrowErrorSet(error, "[%" PRId64 "] Unexpected buffer value %" PRId8, i,
3654
+ view.data.as_int8[i]);
3655
+ return EINVAL;
3656
+ }
3657
+ }
3658
+
3659
+ return NANOARROW_OK;
3660
+ }
3661
+
3662
+ static int ArrowArrayViewValidateFull(struct ArrowArrayView* array_view,
3663
+ struct ArrowError* error) {
3664
+ for (int i = 0; i < NANOARROW_MAX_FIXED_BUFFERS; i++) {
3665
+ switch (array_view->layout.buffer_type[i]) {
3666
+ // Only validate the portion of the buffer that is strictly required,
3667
+ // which includes not validating the offset buffer of a zero-length array.
3668
+ case NANOARROW_BUFFER_TYPE_DATA_OFFSET:
3669
+ if (array_view->length == 0) {
3670
+ continue;
3671
+ }
3672
+ if (array_view->layout.element_size_bits[i] == 32) {
3673
+ struct ArrowBufferView sliced_offsets;
3674
+ sliced_offsets.data.as_int32 =
3675
+ array_view->buffer_views[i].data.as_int32 + array_view->offset;
3676
+ sliced_offsets.size_bytes = (array_view->length + 1) * sizeof(int32_t);
3677
+ NANOARROW_RETURN_NOT_OK(ArrowAssertIncreasingInt32(sliced_offsets, error));
3678
+ } else {
3679
+ struct ArrowBufferView sliced_offsets;
3680
+ sliced_offsets.data.as_int64 =
3681
+ array_view->buffer_views[i].data.as_int64 + array_view->offset;
3682
+ sliced_offsets.size_bytes = (array_view->length + 1) * sizeof(int64_t);
3683
+ NANOARROW_RETURN_NOT_OK(ArrowAssertIncreasingInt64(sliced_offsets, error));
3684
+ }
3685
+ break;
3686
+ default:
3687
+ break;
3688
+ }
3689
+ }
3690
+
3691
+ if (array_view->storage_type == NANOARROW_TYPE_DENSE_UNION ||
3692
+ array_view->storage_type == NANOARROW_TYPE_SPARSE_UNION) {
3693
+ struct ArrowBufferView sliced_type_ids;
3694
+ sliced_type_ids.size_bytes = array_view->length * sizeof(int8_t);
3695
+ if (array_view->length > 0) {
3696
+ sliced_type_ids.data.as_int8 =
3697
+ array_view->buffer_views[0].data.as_int8 + array_view->offset;
3698
+ } else {
3699
+ sliced_type_ids.data.as_int8 = NULL;
3700
+ }
3701
+
3702
+ if (array_view->union_type_id_map == NULL) {
3703
+ // If the union_type_id map is NULL (e.g., when using ArrowArrayInitFromType() +
3704
+ // ArrowArrayAllocateChildren() + ArrowArrayFinishBuilding()), we don't have enough
3705
+ // information to validate this buffer.
3706
+ ArrowErrorSet(error,
3707
+ "Insufficient information provided for validation of union array");
3708
+ return EINVAL;
3709
+ } else if (_ArrowParsedUnionTypeIdsWillEqualChildIndices(
3710
+ array_view->union_type_id_map, array_view->n_children,
3711
+ array_view->n_children)) {
3712
+ NANOARROW_RETURN_NOT_OK(ArrowAssertRangeInt8(
3713
+ sliced_type_ids, 0, (int8_t)(array_view->n_children - 1), error));
3714
+ } else {
3715
+ NANOARROW_RETURN_NOT_OK(ArrowAssertInt8In(sliced_type_ids,
3716
+ array_view->union_type_id_map + 128,
3717
+ array_view->n_children, error));
3718
+ }
3719
+ }
3720
+
3721
+ if (array_view->storage_type == NANOARROW_TYPE_DENSE_UNION &&
3722
+ array_view->union_type_id_map != NULL) {
3723
+ // Check that offsets refer to child elements that actually exist
3724
+ for (int64_t i = 0; i < array_view->length; i++) {
3725
+ int8_t child_id = ArrowArrayViewUnionChildIndex(array_view, i);
3726
+ int64_t offset = ArrowArrayViewUnionChildOffset(array_view, i);
3727
+ int64_t child_length = array_view->children[child_id]->length;
3728
+ if (offset < 0 || offset > child_length) {
3729
+ ArrowErrorSet(error,
3730
+ "[%" PRId64 "] Expected union offset for child id %" PRId8
3731
+ " to be between 0 and %" PRId64
3732
+ " but "
3733
+ "found offset value %" PRId64,
3734
+ i, child_id, child_length, offset);
3735
+ return EINVAL;
3736
+ }
3737
+ }
3738
+ }
3739
+
3740
+ if (array_view->storage_type == NANOARROW_TYPE_RUN_END_ENCODED) {
3741
+ struct ArrowArrayView* run_ends_view = array_view->children[0];
3742
+ if (run_ends_view->length > 0) {
3743
+ int64_t last_run_end = ArrowArrayViewGetIntUnsafe(run_ends_view, 0);
3744
+ for (int64_t i = 1; i < run_ends_view->length; i++) {
3745
+ const int64_t run_end = ArrowArrayViewGetIntUnsafe(run_ends_view, i);
3746
+ if (run_end <= last_run_end) {
3747
+ ArrowErrorSet(
3748
+ error,
3749
+ "Every run end must be strictly greater than the previous run end, "
3750
+ "but run_ends[%" PRId64 " is %" PRId64 " and run_ends[%" PRId64
3751
+ "] is %" PRId64,
3752
+ i, run_end, i - 1, last_run_end);
3753
+ return EINVAL;
3754
+ }
3755
+ last_run_end = run_end;
3756
+ }
3757
+ }
3758
+ }
3759
+
3760
+ if (array_view->storage_type == NANOARROW_TYPE_LIST_VIEW ||
3761
+ array_view->storage_type == NANOARROW_TYPE_LARGE_LIST_VIEW) {
3762
+ int64_t child_len = array_view->children[0]->length;
3763
+
3764
+ struct ArrowBufferView offsets, sizes;
3765
+ offsets.data.data = array_view->buffer_views[1].data.data;
3766
+ sizes.data.data = array_view->buffer_views[2].data.data;
3767
+
3768
+ for (int64_t i = array_view->offset; i < array_view->length + array_view->offset;
3769
+ i++) {
3770
+ int64_t offset, size;
3771
+ if (array_view->storage_type == NANOARROW_TYPE_LIST_VIEW) {
3772
+ offset = offsets.data.as_int32[i];
3773
+ size = sizes.data.as_int32[i];
3774
+ } else {
3775
+ offset = offsets.data.as_int64[i];
3776
+ size = sizes.data.as_int64[i];
3777
+ }
3778
+
3779
+ if (offset < 0) {
3780
+ ArrowErrorSet(error, "Invalid negative offset %" PRId64 " at index %" PRId64,
3781
+ offset, i);
3782
+ return EINVAL;
3783
+ }
3784
+
3785
+ if (size < 0) {
3786
+ ArrowErrorSet(error, "Invalid negative size %" PRId64 " at index %" PRId64, size,
3787
+ i);
3788
+ return EINVAL;
3789
+ }
3790
+
3791
+ if ((offset + size) > child_len) {
3792
+ ArrowErrorSet(error,
3793
+ "Offset: %" PRId64 " + size: %" PRId64 " at index: %" PRId64
3794
+ " exceeds length of child view: %" PRId64,
3795
+ offset, size, i, child_len);
3796
+ return EINVAL;
3797
+ }
3798
+ }
3799
+ }
3800
+
3801
+ // Recurse for children
3802
+ for (int64_t i = 0; i < array_view->n_children; i++) {
3803
+ NANOARROW_RETURN_NOT_OK(ArrowArrayViewValidateFull(array_view->children[i], error));
3804
+ }
3805
+
3806
+ // Dictionary validation not implemented
3807
+ if (array_view->dictionary != NULL) {
3808
+ NANOARROW_RETURN_NOT_OK(ArrowArrayViewValidateFull(array_view->dictionary, error));
3809
+ // TODO: validate the indices
3810
+ }
3811
+
3812
+ return NANOARROW_OK;
3813
+ }
3814
+
3815
+ ArrowErrorCode ArrowArrayViewValidate(struct ArrowArrayView* array_view,
3816
+ enum ArrowValidationLevel validation_level,
3817
+ struct ArrowError* error) {
3818
+ switch (validation_level) {
3819
+ case NANOARROW_VALIDATION_LEVEL_NONE:
3820
+ return NANOARROW_OK;
3821
+ case NANOARROW_VALIDATION_LEVEL_MINIMAL:
3822
+ return ArrowArrayViewValidateMinimal(array_view, error);
3823
+ case NANOARROW_VALIDATION_LEVEL_DEFAULT:
3824
+ return ArrowArrayViewValidateDefault(array_view, error);
3825
+ case NANOARROW_VALIDATION_LEVEL_FULL:
3826
+ NANOARROW_RETURN_NOT_OK(ArrowArrayViewValidateDefault(array_view, error));
3827
+ return ArrowArrayViewValidateFull(array_view, error);
3828
+ }
3829
+
3830
+ ArrowErrorSet(error, "validation_level not recognized");
3831
+ return EINVAL;
3832
+ }
3833
+
3834
+ struct ArrowComparisonInternalState {
3835
+ enum ArrowCompareLevel level;
3836
+ int is_equal;
3837
+ struct ArrowError* reason;
3838
+ };
3839
+
3840
+ NANOARROW_CHECK_PRINTF_ATTRIBUTE static void ArrowComparePrependPath(
3841
+ struct ArrowError* out, const char* fmt, ...) {
3842
+ if (out == NULL) {
3843
+ return;
3844
+ }
3845
+
3846
+ char prefix[128];
3847
+ prefix[0] = '\0';
3848
+ va_list args;
3849
+ va_start(args, fmt);
3850
+ int prefix_len = vsnprintf(prefix, sizeof(prefix), fmt, args);
3851
+ va_end(args);
3852
+
3853
+ if (prefix_len <= 0) {
3854
+ return;
3855
+ }
3856
+
3857
+ size_t out_len = strlen(out->message);
3858
+ size_t out_len_to_move = sizeof(struct ArrowError) - prefix_len - 1;
3859
+ if (out_len_to_move > out_len) {
3860
+ out_len_to_move = out_len;
3861
+ }
3862
+
3863
+ memmove(out->message + prefix_len, out->message, out_len_to_move);
3864
+ memcpy(out->message, prefix, prefix_len);
3865
+ out->message[out_len + prefix_len] = '\0';
3866
+ }
3867
+
3868
+ #define SET_NOT_EQUAL_AND_RETURN_IF_IMPL(cond_, state_, reason_) \
3869
+ do { \
3870
+ if (cond_) { \
3871
+ ArrowErrorSet(state_->reason, ": %s", reason_); \
3872
+ state_->is_equal = 0; \
3873
+ return; \
3874
+ } \
3875
+ } while (0)
3876
+
3877
+ #define SET_NOT_EQUAL_AND_RETURN_IF(condition_, state_) \
3878
+ SET_NOT_EQUAL_AND_RETURN_IF_IMPL(condition_, state_, #condition_)
3879
+
3880
+ static void ArrowArrayViewCompareBuffer(const struct ArrowArrayView* actual,
3881
+ const struct ArrowArrayView* expected, int i,
3882
+ struct ArrowComparisonInternalState* state) {
3883
+ SET_NOT_EQUAL_AND_RETURN_IF(
3884
+ actual->buffer_views[i].size_bytes != expected->buffer_views[i].size_bytes, state);
3885
+
3886
+ int64_t buffer_size = actual->buffer_views[i].size_bytes;
3887
+ if (buffer_size > 0) {
3888
+ SET_NOT_EQUAL_AND_RETURN_IF(
3889
+ memcmp(actual->buffer_views[i].data.data, expected->buffer_views[i].data.data,
3890
+ buffer_size) != 0,
3891
+ state);
3892
+ }
3893
+ }
3894
+
3895
+ static void ArrowArrayViewCompareIdentical(const struct ArrowArrayView* actual,
3896
+ const struct ArrowArrayView* expected,
3897
+ struct ArrowComparisonInternalState* state) {
3898
+ SET_NOT_EQUAL_AND_RETURN_IF(actual->storage_type != expected->storage_type, state);
3899
+ SET_NOT_EQUAL_AND_RETURN_IF(actual->n_children != expected->n_children, state);
3900
+ SET_NOT_EQUAL_AND_RETURN_IF(actual->dictionary == NULL && expected->dictionary != NULL,
3901
+ state);
3902
+ SET_NOT_EQUAL_AND_RETURN_IF(actual->dictionary != NULL && expected->dictionary == NULL,
3903
+ state);
3904
+
3905
+ SET_NOT_EQUAL_AND_RETURN_IF(actual->length != expected->length, state);
3906
+ SET_NOT_EQUAL_AND_RETURN_IF(actual->offset != expected->offset, state);
3907
+ SET_NOT_EQUAL_AND_RETURN_IF(actual->null_count != expected->null_count, state);
3908
+
3909
+ for (int i = 0; i < NANOARROW_MAX_FIXED_BUFFERS; i++) {
3910
+ ArrowArrayViewCompareBuffer(actual, expected, i, state);
3911
+ if (!state->is_equal) {
3912
+ ArrowComparePrependPath(state->reason, ".buffers[%d]", i);
3913
+ return;
3914
+ }
3915
+ }
3916
+
3917
+ for (int64_t i = 0; i < actual->n_children; i++) {
3918
+ ArrowArrayViewCompareIdentical(actual->children[i], expected->children[i], state);
3919
+ if (!state->is_equal) {
3920
+ ArrowComparePrependPath(state->reason, ".children[%" PRId64 "]", i);
3921
+ return;
3922
+ }
3923
+ }
3924
+
3925
+ if (actual->dictionary != NULL) {
3926
+ ArrowArrayViewCompareIdentical(actual->dictionary, expected->dictionary, state);
3927
+ if (!state->is_equal) {
3928
+ ArrowComparePrependPath(state->reason, ".dictionary");
3929
+ return;
3930
+ }
3931
+ }
3932
+ }
3933
+
3934
+ // Top-level entry point to take care of creating, cleaning up, and
3935
+ // propagating the ArrowComparisonInternalState to the caller
3936
+ ArrowErrorCode ArrowArrayViewCompare(const struct ArrowArrayView* actual,
3937
+ const struct ArrowArrayView* expected,
3938
+ enum ArrowCompareLevel level, int* out,
3939
+ struct ArrowError* reason) {
3940
+ struct ArrowComparisonInternalState state;
3941
+ state.level = level;
3942
+ state.is_equal = 1;
3943
+ state.reason = reason;
3944
+
3945
+ switch (level) {
3946
+ case NANOARROW_COMPARE_IDENTICAL:
3947
+ ArrowArrayViewCompareIdentical(actual, expected, &state);
3948
+ break;
3949
+ default:
3950
+ return EINVAL;
3951
+ }
3952
+
3953
+ *out = state.is_equal;
3954
+ if (!state.is_equal) {
3955
+ ArrowComparePrependPath(state.reason, "root");
3956
+ }
3957
+
3958
+ return NANOARROW_OK;
3959
+ }
3960
+
3961
+ #undef SET_NOT_EQUAL_AND_RETURN_IF
3962
+ #undef SET_NOT_EQUAL_AND_RETURN_IF_IMPL
3963
+ // Licensed to the Apache Software Foundation (ASF) under one
3964
+ // or more contributor license agreements. See the NOTICE file
3965
+ // distributed with this work for additional information
3966
+ // regarding copyright ownership. The ASF licenses this file
3967
+ // to you under the Apache License, Version 2.0 (the
3968
+ // "License"); you may not use this file except in compliance
3969
+ // with the License. You may obtain a copy of the License at
3970
+ //
3971
+ // http://www.apache.org/licenses/LICENSE-2.0
3972
+ //
3973
+ // Unless required by applicable law or agreed to in writing,
3974
+ // software distributed under the License is distributed on an
3975
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
3976
+ // KIND, either express or implied. See the License for the
3977
+ // specific language governing permissions and limitations
3978
+ // under the License.
3979
+
3980
+ #include <errno.h>
3981
+
3982
+ #include "nanoarrow/nanoarrow.h"
3983
+
3984
+ struct BasicArrayStreamPrivate {
3985
+ struct ArrowSchema schema;
3986
+ int64_t n_arrays;
3987
+ struct ArrowArray* arrays;
3988
+ int64_t arrays_i;
3989
+ };
3990
+
3991
+ static int ArrowBasicArrayStreamGetSchema(struct ArrowArrayStream* array_stream,
3992
+ struct ArrowSchema* schema) {
3993
+ if (array_stream == NULL || array_stream->release == NULL) {
3994
+ return EINVAL;
3995
+ }
3996
+
3997
+ struct BasicArrayStreamPrivate* private_data =
3998
+ (struct BasicArrayStreamPrivate*)array_stream->private_data;
3999
+ return ArrowSchemaDeepCopy(&private_data->schema, schema);
4000
+ }
4001
+
4002
+ static int ArrowBasicArrayStreamGetNext(struct ArrowArrayStream* array_stream,
4003
+ struct ArrowArray* array) {
4004
+ if (array_stream == NULL || array_stream->release == NULL) {
4005
+ return EINVAL;
4006
+ }
4007
+
4008
+ struct BasicArrayStreamPrivate* private_data =
4009
+ (struct BasicArrayStreamPrivate*)array_stream->private_data;
4010
+
4011
+ if (private_data->arrays_i == private_data->n_arrays) {
4012
+ array->release = NULL;
4013
+ return NANOARROW_OK;
4014
+ }
4015
+
4016
+ ArrowArrayMove(&private_data->arrays[private_data->arrays_i++], array);
4017
+ return NANOARROW_OK;
4018
+ }
4019
+
4020
+ static const char* ArrowBasicArrayStreamGetLastError(
4021
+ struct ArrowArrayStream* array_stream) {
4022
+ NANOARROW_UNUSED(array_stream);
4023
+ return NULL;
4024
+ }
4025
+
4026
+ static void ArrowBasicArrayStreamRelease(struct ArrowArrayStream* array_stream) {
4027
+ if (array_stream == NULL || array_stream->release == NULL) {
4028
+ return;
4029
+ }
4030
+
4031
+ struct BasicArrayStreamPrivate* private_data =
4032
+ (struct BasicArrayStreamPrivate*)array_stream->private_data;
4033
+
4034
+ if (private_data->schema.release != NULL) {
4035
+ ArrowSchemaRelease(&private_data->schema);
4036
+ }
4037
+
4038
+ for (int64_t i = 0; i < private_data->n_arrays; i++) {
4039
+ if (private_data->arrays[i].release != NULL) {
4040
+ ArrowArrayRelease(&private_data->arrays[i]);
4041
+ }
4042
+ }
4043
+
4044
+ if (private_data->arrays != NULL) {
4045
+ ArrowFree(private_data->arrays);
4046
+ }
4047
+
4048
+ ArrowFree(private_data);
4049
+ array_stream->release = NULL;
4050
+ }
4051
+
4052
+ ArrowErrorCode ArrowBasicArrayStreamInit(struct ArrowArrayStream* array_stream,
4053
+ struct ArrowSchema* schema, int64_t n_arrays) {
4054
+ struct BasicArrayStreamPrivate* private_data =
4055
+ (struct BasicArrayStreamPrivate*)ArrowMalloc(
4056
+ sizeof(struct BasicArrayStreamPrivate));
4057
+ if (private_data == NULL) {
4058
+ return ENOMEM;
4059
+ }
4060
+
4061
+ ArrowSchemaMove(schema, &private_data->schema);
4062
+
4063
+ private_data->n_arrays = n_arrays;
4064
+ private_data->arrays = NULL;
4065
+ private_data->arrays_i = 0;
4066
+
4067
+ if (n_arrays > 0) {
4068
+ private_data->arrays =
4069
+ (struct ArrowArray*)ArrowMalloc(n_arrays * sizeof(struct ArrowArray));
4070
+ if (private_data->arrays == NULL) {
4071
+ ArrowBasicArrayStreamRelease(array_stream);
4072
+ ArrowFree(private_data);
4073
+ return ENOMEM;
4074
+ }
4075
+ }
4076
+
4077
+ for (int64_t i = 0; i < private_data->n_arrays; i++) {
4078
+ private_data->arrays[i].release = NULL;
4079
+ }
4080
+
4081
+ array_stream->get_schema = &ArrowBasicArrayStreamGetSchema;
4082
+ array_stream->get_next = &ArrowBasicArrayStreamGetNext;
4083
+ array_stream->get_last_error = ArrowBasicArrayStreamGetLastError;
4084
+ array_stream->release = ArrowBasicArrayStreamRelease;
4085
+ array_stream->private_data = private_data;
4086
+ return NANOARROW_OK;
4087
+ }
4088
+
4089
+ void ArrowBasicArrayStreamSetArray(struct ArrowArrayStream* array_stream, int64_t i,
4090
+ struct ArrowArray* array) {
4091
+ struct BasicArrayStreamPrivate* private_data =
4092
+ (struct BasicArrayStreamPrivate*)array_stream->private_data;
4093
+ ArrowArrayMove(array, &private_data->arrays[i]);
4094
+ }
4095
+
4096
+ ArrowErrorCode ArrowBasicArrayStreamValidate(const struct ArrowArrayStream* array_stream,
4097
+ struct ArrowError* error) {
4098
+ struct BasicArrayStreamPrivate* private_data =
4099
+ (struct BasicArrayStreamPrivate*)array_stream->private_data;
4100
+
4101
+ struct ArrowArrayView array_view;
4102
+ NANOARROW_RETURN_NOT_OK(
4103
+ ArrowArrayViewInitFromSchema(&array_view, &private_data->schema, error));
4104
+
4105
+ for (int64_t i = 0; i < private_data->n_arrays; i++) {
4106
+ if (private_data->arrays[i].release != NULL) {
4107
+ int result = ArrowArrayViewSetArray(&array_view, &private_data->arrays[i], error);
4108
+ if (result != NANOARROW_OK) {
4109
+ ArrowArrayViewReset(&array_view);
4110
+ return result;
4111
+ }
4112
+ }
4113
+ }
4114
+
4115
+ ArrowArrayViewReset(&array_view);
4116
+ return NANOARROW_OK;
4117
+ }