red-arrow-format 23.0.1 → 25.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile +7 -2
  3. data/Rakefile +23 -1
  4. data/lib/arrow-format/array.rb +486 -117
  5. data/lib/arrow-format/bitmap.rb +25 -6
  6. data/lib/arrow-format/buffer-alignable.rb +35 -0
  7. data/lib/arrow-format/dictionary.rb +29 -0
  8. data/lib/arrow-format/error.rb +1 -1
  9. data/lib/arrow-format/field.rb +24 -3
  10. data/lib/arrow-format/file-reader.rb +47 -25
  11. data/lib/arrow-format/file-writer.rb +56 -0
  12. data/lib/arrow-format/flatbuffers.rb +67 -0
  13. data/lib/arrow-format/org/apache/arrow/flatbuf/binary.rb +5 -1
  14. data/lib/arrow-format/org/apache/arrow/flatbuf/binary_view.rb +5 -1
  15. data/lib/arrow-format/org/apache/arrow/flatbuf/block.rb +10 -2
  16. data/lib/arrow-format/org/apache/arrow/flatbuf/body_compression.rb +8 -1
  17. data/lib/arrow-format/org/apache/arrow/flatbuf/body_compression_method.rb +3 -1
  18. data/lib/arrow-format/org/apache/arrow/flatbuf/bool.rb +5 -1
  19. data/lib/arrow-format/org/apache/arrow/flatbuf/buffer.rb +8 -1
  20. data/lib/arrow-format/org/apache/arrow/flatbuf/compression_type.rb +1 -1
  21. data/lib/arrow-format/org/apache/arrow/flatbuf/date.rb +7 -1
  22. data/lib/arrow-format/org/apache/arrow/flatbuf/date_unit.rb +1 -1
  23. data/lib/arrow-format/org/apache/arrow/flatbuf/decimal.rb +9 -1
  24. data/lib/arrow-format/org/apache/arrow/flatbuf/dictionary_batch.rb +9 -1
  25. data/lib/arrow-format/org/apache/arrow/flatbuf/dictionary_encoding.rb +11 -2
  26. data/lib/arrow-format/org/apache/arrow/flatbuf/dictionary_kind.rb +1 -1
  27. data/lib/arrow-format/org/apache/arrow/flatbuf/duration.rb +7 -1
  28. data/lib/arrow-format/org/apache/arrow/flatbuf/endianness.rb +1 -1
  29. data/lib/arrow-format/org/apache/arrow/flatbuf/feature.rb +1 -1
  30. data/lib/arrow-format/org/apache/arrow/flatbuf/field.rb +13 -1
  31. data/lib/arrow-format/org/apache/arrow/flatbuf/field_node.rb +8 -1
  32. data/lib/arrow-format/org/apache/arrow/flatbuf/fixed_size_binary.rb +7 -1
  33. data/lib/arrow-format/org/apache/arrow/flatbuf/fixed_size_list.rb +7 -1
  34. data/lib/arrow-format/org/apache/arrow/flatbuf/floating_point.rb +7 -1
  35. data/lib/arrow-format/org/apache/arrow/flatbuf/footer.rb +22 -2
  36. data/lib/arrow-format/org/apache/arrow/flatbuf/int.rb +8 -1
  37. data/lib/arrow-format/org/apache/arrow/flatbuf/interval.rb +7 -1
  38. data/lib/arrow-format/org/apache/arrow/flatbuf/interval_unit.rb +1 -1
  39. data/lib/arrow-format/org/apache/arrow/flatbuf/key_value.rb +8 -1
  40. data/lib/arrow-format/org/apache/arrow/flatbuf/large_binary.rb +5 -1
  41. data/lib/arrow-format/org/apache/arrow/flatbuf/large_list.rb +5 -1
  42. data/lib/arrow-format/org/apache/arrow/flatbuf/large_list_view.rb +5 -1
  43. data/lib/arrow-format/org/apache/arrow/flatbuf/large_utf8.rb +5 -1
  44. data/lib/arrow-format/org/apache/arrow/flatbuf/list.rb +5 -1
  45. data/lib/arrow-format/org/apache/arrow/flatbuf/list_view.rb +5 -1
  46. data/lib/arrow-format/org/apache/arrow/flatbuf/map.rb +8 -2
  47. data/lib/arrow-format/org/apache/arrow/flatbuf/message.rb +22 -2
  48. data/lib/arrow-format/org/apache/arrow/flatbuf/message_header.rb +1 -1
  49. data/lib/arrow-format/org/apache/arrow/flatbuf/metadata_version.rb +1 -1
  50. data/lib/arrow-format/org/apache/arrow/flatbuf/null.rb +5 -1
  51. data/lib/arrow-format/org/apache/arrow/flatbuf/precision.rb +1 -1
  52. data/lib/arrow-format/org/apache/arrow/flatbuf/record_batch.rb +12 -2
  53. data/lib/arrow-format/org/apache/arrow/flatbuf/run_end_encoded.rb +5 -1
  54. data/lib/arrow-format/org/apache/arrow/flatbuf/schema.rb +10 -1
  55. data/lib/arrow-format/org/apache/arrow/flatbuf/sparse_matrix_compressed_axis.rb +1 -1
  56. data/lib/arrow-format/org/apache/arrow/flatbuf/sparse_matrix_index_csx.rb +12 -2
  57. data/lib/arrow-format/org/apache/arrow/flatbuf/sparse_tensor.rb +13 -1
  58. data/lib/arrow-format/org/apache/arrow/flatbuf/sparse_tensor_index.rb +1 -1
  59. data/lib/arrow-format/org/apache/arrow/flatbuf/sparse_tensor_index_coo.rb +13 -4
  60. data/lib/arrow-format/org/apache/arrow/flatbuf/sparse_tensor_index_csf.rb +14 -4
  61. data/lib/arrow-format/org/apache/arrow/flatbuf/struct_.rb +5 -1
  62. data/lib/arrow-format/org/apache/arrow/flatbuf/tensor.rb +11 -1
  63. data/lib/arrow-format/org/apache/arrow/flatbuf/tensor_dim.rb +8 -1
  64. data/lib/arrow-format/org/apache/arrow/flatbuf/time.rb +8 -1
  65. data/lib/arrow-format/org/apache/arrow/flatbuf/time_unit.rb +1 -1
  66. data/lib/arrow-format/org/apache/arrow/flatbuf/timestamp.rb +8 -1
  67. data/lib/arrow-format/org/apache/arrow/flatbuf/type.rb +1 -1
  68. data/lib/arrow-format/org/apache/arrow/flatbuf/union.rb +8 -1
  69. data/lib/arrow-format/org/apache/arrow/flatbuf/union_mode.rb +1 -1
  70. data/lib/arrow-format/org/apache/arrow/flatbuf/utf8.rb +5 -1
  71. data/lib/arrow-format/org/apache/arrow/flatbuf/utf8view.rb +5 -1
  72. data/lib/arrow-format/readable.rb +151 -96
  73. data/lib/arrow-format/record-batch.rb +70 -1
  74. data/lib/arrow-format/schema.rb +14 -1
  75. data/lib/arrow-format/streaming-pull-reader.rb +49 -23
  76. data/lib/arrow-format/streaming-reader.rb +51 -14
  77. data/lib/arrow-format/streaming-writer.rb +156 -0
  78. data/lib/arrow-format/type.rb +397 -34
  79. data/lib/arrow-format/version.rb +1 -1
  80. data/lib/arrow-format.rb +2 -0
  81. data/red-arrow-format.gemspec +2 -1
  82. metadata +9 -4
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 521a7a4b398d999c634cb3cc006608a94af7b2f7848c50c30c34273d174b7056
4
- data.tar.gz: c4c945e615bc5307bb05f6ae8c236bb0c2de2bfe513fb8420f1bd05c402eb22e
3
+ metadata.gz: 4113dbcda2378ab7d587d206bb50d89ebd6326b45155ca1ae84aa1db2c93c1f8
4
+ data.tar.gz: de65113d15c44830aa40b6699b02c1844f162fb52139542947b31c5c4b034bc9
5
5
  SHA512:
6
- metadata.gz: 545968a5e4117dfb171151278f4ecf2ab649390e8d805f0e9e6bebbfd229422773ff7c9a5da0b56328f07aa469203fbe91c3065189d3ea3d0499724cfbd8ac6b
7
- data.tar.gz: 7b0a042a7d36fea98aa7c3a78e62c1a98c841ada5c97eb28b4ae0ac37585e93dcf8b1bea91645b1e66d8c42335d8ec503246d1ca87e4dbab15a5fc43c277853f
6
+ metadata.gz: 217422c5e7a43abe12826fea4ebfffdce96d7f97b4659f861c08924c059455be73d638e5b9bfae081b7110207524f2d5351fd4b3b03defe5077b907ff1eeae60
7
+ data.tar.gz: 67801cd0f4a81027f832b3b6b68d045d5f2ec92632b456f0188a448fb706dd6468339753b563e69f71ee1ebd6cfea68a72d87cf5968a556a0c35d3647ba88da8
data/Gemfile CHANGED
@@ -21,6 +21,11 @@ source "https://rubygems.org/"
21
21
 
22
22
  gemspec
23
23
 
24
- gem "rake"
25
24
  gem "red-arrow", path: "../red-arrow"
26
- gem "test-unit"
25
+
26
+ group :development do
27
+ gem "benchmark-driver"
28
+ gem "rake"
29
+ gem "stringio"
30
+ gem "test-unit"
31
+ end
data/Rakefile CHANGED
@@ -39,7 +39,29 @@ task :test do
39
39
  end
40
40
  end
41
41
 
42
- namespace :flatbuffers do
42
+ benchmark_tasks = []
43
+ namespace :benchmark do
44
+ Dir.glob("benchmark/*.yaml").sort.each do |yaml|
45
+ name = File.basename(yaml, ".*")
46
+ command_line = [
47
+ RbConfig.ruby, "-v", "-S", "benchmark-driver", File.expand_path(yaml),
48
+ ]
49
+
50
+ desc "Run #{name} benchmark"
51
+ task name do
52
+ puts("```console")
53
+ puts("$ #{command_line.join(" ")}")
54
+ sh(*command_line, verbose: false)
55
+ puts("```")
56
+ end
57
+ benchmark_tasks << "benchmark:#{name}"
58
+ end
59
+ end
60
+
61
+ desc "Run all benchmarks"
62
+ task :benchmark => benchmark_tasks
63
+
64
+ namespace :flat_buffers do
43
65
  desc "Generate FlatBuffers code"
44
66
  task :generate do
45
67
  Dir.mktmpdir do |tmp_dir|