red-arrow-format 23.0.0 → 24.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile +7 -2
  3. data/NOTICE.txt +1 -1
  4. data/Rakefile +23 -1
  5. data/lib/arrow-format/array.rb +486 -117
  6. data/lib/arrow-format/bitmap.rb +25 -6
  7. data/lib/arrow-format/buffer-alignable.rb +35 -0
  8. data/lib/arrow-format/dictionary.rb +29 -0
  9. data/lib/arrow-format/error.rb +1 -1
  10. data/lib/arrow-format/field.rb +24 -3
  11. data/lib/arrow-format/file-reader.rb +47 -25
  12. data/lib/arrow-format/file-writer.rb +56 -0
  13. data/lib/arrow-format/flatbuffers.rb +67 -0
  14. data/lib/arrow-format/org/apache/arrow/flatbuf/binary.rb +5 -1
  15. data/lib/arrow-format/org/apache/arrow/flatbuf/binary_view.rb +5 -1
  16. data/lib/arrow-format/org/apache/arrow/flatbuf/block.rb +10 -2
  17. data/lib/arrow-format/org/apache/arrow/flatbuf/body_compression.rb +8 -1
  18. data/lib/arrow-format/org/apache/arrow/flatbuf/body_compression_method.rb +3 -1
  19. data/lib/arrow-format/org/apache/arrow/flatbuf/bool.rb +5 -1
  20. data/lib/arrow-format/org/apache/arrow/flatbuf/buffer.rb +8 -1
  21. data/lib/arrow-format/org/apache/arrow/flatbuf/compression_type.rb +1 -1
  22. data/lib/arrow-format/org/apache/arrow/flatbuf/date.rb +7 -1
  23. data/lib/arrow-format/org/apache/arrow/flatbuf/date_unit.rb +1 -1
  24. data/lib/arrow-format/org/apache/arrow/flatbuf/decimal.rb +9 -1
  25. data/lib/arrow-format/org/apache/arrow/flatbuf/dictionary_batch.rb +9 -1
  26. data/lib/arrow-format/org/apache/arrow/flatbuf/dictionary_encoding.rb +11 -2
  27. data/lib/arrow-format/org/apache/arrow/flatbuf/dictionary_kind.rb +1 -1
  28. data/lib/arrow-format/org/apache/arrow/flatbuf/duration.rb +7 -1
  29. data/lib/arrow-format/org/apache/arrow/flatbuf/endianness.rb +1 -1
  30. data/lib/arrow-format/org/apache/arrow/flatbuf/feature.rb +1 -1
  31. data/lib/arrow-format/org/apache/arrow/flatbuf/field.rb +13 -1
  32. data/lib/arrow-format/org/apache/arrow/flatbuf/field_node.rb +8 -1
  33. data/lib/arrow-format/org/apache/arrow/flatbuf/fixed_size_binary.rb +7 -1
  34. data/lib/arrow-format/org/apache/arrow/flatbuf/fixed_size_list.rb +7 -1
  35. data/lib/arrow-format/org/apache/arrow/flatbuf/floating_point.rb +7 -1
  36. data/lib/arrow-format/org/apache/arrow/flatbuf/footer.rb +22 -2
  37. data/lib/arrow-format/org/apache/arrow/flatbuf/int.rb +8 -1
  38. data/lib/arrow-format/org/apache/arrow/flatbuf/interval.rb +7 -1
  39. data/lib/arrow-format/org/apache/arrow/flatbuf/interval_unit.rb +1 -1
  40. data/lib/arrow-format/org/apache/arrow/flatbuf/key_value.rb +8 -1
  41. data/lib/arrow-format/org/apache/arrow/flatbuf/large_binary.rb +5 -1
  42. data/lib/arrow-format/org/apache/arrow/flatbuf/large_list.rb +5 -1
  43. data/lib/arrow-format/org/apache/arrow/flatbuf/large_list_view.rb +5 -1
  44. data/lib/arrow-format/org/apache/arrow/flatbuf/large_utf8.rb +5 -1
  45. data/lib/arrow-format/org/apache/arrow/flatbuf/list.rb +5 -1
  46. data/lib/arrow-format/org/apache/arrow/flatbuf/list_view.rb +5 -1
  47. data/lib/arrow-format/org/apache/arrow/flatbuf/map.rb +8 -2
  48. data/lib/arrow-format/org/apache/arrow/flatbuf/message.rb +22 -2
  49. data/lib/arrow-format/org/apache/arrow/flatbuf/message_header.rb +1 -1
  50. data/lib/arrow-format/org/apache/arrow/flatbuf/metadata_version.rb +1 -1
  51. data/lib/arrow-format/org/apache/arrow/flatbuf/null.rb +5 -1
  52. data/lib/arrow-format/org/apache/arrow/flatbuf/precision.rb +1 -1
  53. data/lib/arrow-format/org/apache/arrow/flatbuf/record_batch.rb +12 -2
  54. data/lib/arrow-format/org/apache/arrow/flatbuf/run_end_encoded.rb +5 -1
  55. data/lib/arrow-format/org/apache/arrow/flatbuf/schema.rb +10 -1
  56. data/lib/arrow-format/org/apache/arrow/flatbuf/sparse_matrix_compressed_axis.rb +1 -1
  57. data/lib/arrow-format/org/apache/arrow/flatbuf/sparse_matrix_index_csx.rb +12 -2
  58. data/lib/arrow-format/org/apache/arrow/flatbuf/sparse_tensor.rb +13 -1
  59. data/lib/arrow-format/org/apache/arrow/flatbuf/sparse_tensor_index.rb +1 -1
  60. data/lib/arrow-format/org/apache/arrow/flatbuf/sparse_tensor_index_coo.rb +13 -4
  61. data/lib/arrow-format/org/apache/arrow/flatbuf/sparse_tensor_index_csf.rb +14 -4
  62. data/lib/arrow-format/org/apache/arrow/flatbuf/struct_.rb +5 -1
  63. data/lib/arrow-format/org/apache/arrow/flatbuf/tensor.rb +11 -1
  64. data/lib/arrow-format/org/apache/arrow/flatbuf/tensor_dim.rb +8 -1
  65. data/lib/arrow-format/org/apache/arrow/flatbuf/time.rb +8 -1
  66. data/lib/arrow-format/org/apache/arrow/flatbuf/time_unit.rb +1 -1
  67. data/lib/arrow-format/org/apache/arrow/flatbuf/timestamp.rb +8 -1
  68. data/lib/arrow-format/org/apache/arrow/flatbuf/type.rb +1 -1
  69. data/lib/arrow-format/org/apache/arrow/flatbuf/union.rb +8 -1
  70. data/lib/arrow-format/org/apache/arrow/flatbuf/union_mode.rb +1 -1
  71. data/lib/arrow-format/org/apache/arrow/flatbuf/utf8.rb +5 -1
  72. data/lib/arrow-format/org/apache/arrow/flatbuf/utf8view.rb +5 -1
  73. data/lib/arrow-format/readable.rb +151 -96
  74. data/lib/arrow-format/record-batch.rb +70 -1
  75. data/lib/arrow-format/schema.rb +14 -1
  76. data/lib/arrow-format/streaming-pull-reader.rb +49 -23
  77. data/lib/arrow-format/streaming-reader.rb +51 -14
  78. data/lib/arrow-format/streaming-writer.rb +156 -0
  79. data/lib/arrow-format/type.rb +397 -34
  80. data/lib/arrow-format/version.rb +1 -1
  81. data/lib/arrow-format.rb +2 -0
  82. data/red-arrow-format.gemspec +2 -1
  83. metadata +9 -4
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 32a1d4d28ebd16699f70de7c1738dd54f98e1811caaf649b648f8f6ca000c762
4
- data.tar.gz: 30ae839893a1a3d8ca08992089248ddb2cd286415b5e930afa059883bb5b6fdb
3
+ metadata.gz: d75e73a10c7e465e4dc0fe2b1b4f156fa00360e0b098e7e21c012c82991b424c
4
+ data.tar.gz: 42bbd4355bc95ab574f5de81dba631911ba54252b28656c2c8a58762830d0db6
5
5
  SHA512:
6
- metadata.gz: fe4c209c7fbd9c565ac71678564f425589375bad7d5048cc5d646820f59366a6d9147a5aa3ecc661a3823d871710059787aa34a1dc36a916f2cc5ebfbba69c07
7
- data.tar.gz: 578f0cbe2b59a91791844aadb91d0583520b7f2522ca18fe982f97b565376dd9db2c4acc715b48a3fb64cc26ccaa3b4621d6a14a0c1f90e8b7a58ad9214ab338
6
+ metadata.gz: 13dd0f2905f676a7dc4f6d2928a8d43f99c93e30c39edcbe826b9ae2c2940717eab23a5966bcb72e25ed63306bfbd281c406cc083289d99fb9fb1709891cbd22
7
+ data.tar.gz: 71f64457736369361f74ca73adf8e69ebcbcdd015ccc04e1add8903c522cd0fa977a45495eb3d4c240cecafd0ecfece76f883c3929d4fa4d0100ea3b9ef30416
data/Gemfile CHANGED
@@ -21,6 +21,11 @@ source "https://rubygems.org/"
21
21
 
22
22
  gemspec
23
23
 
24
- gem "rake"
25
24
  gem "red-arrow", path: "../red-arrow"
26
- gem "test-unit"
25
+
26
+ group :development do
27
+ gem "benchmark-driver"
28
+ gem "rake"
29
+ gem "stringio"
30
+ gem "test-unit"
31
+ end
data/NOTICE.txt CHANGED
@@ -1,2 +1,2 @@
1
1
  Apache Arrow
2
- Copyright 2025 The Apache Software Foundation
2
+ Copyright 2025-2026 The Apache Software Foundation
data/Rakefile CHANGED
@@ -39,7 +39,29 @@ task :test do
39
39
  end
40
40
  end
41
41
 
42
- namespace :flatbuffers do
42
+ benchmark_tasks = []
43
+ namespace :benchmark do
44
+ Dir.glob("benchmark/*.yaml").sort.each do |yaml|
45
+ name = File.basename(yaml, ".*")
46
+ command_line = [
47
+ RbConfig.ruby, "-v", "-S", "benchmark-driver", File.expand_path(yaml),
48
+ ]
49
+
50
+ desc "Run #{name} benchmark"
51
+ task name do
52
+ puts("```console")
53
+ puts("$ #{command_line.join(" ")}")
54
+ sh(*command_line, verbose: false)
55
+ puts("```")
56
+ end
57
+ benchmark_tasks << "benchmark:#{name}"
58
+ end
59
+ end
60
+
61
+ desc "Run all benchmarks"
62
+ task :benchmark => benchmark_tasks
63
+
64
+ namespace :flat_buffers do
43
65
  desc "Generate FlatBuffers code"
44
66
  task :generate do
45
67
  Dir.mktmpdir do |tmp_dir|